{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim.parsing.preprocessing import *\n",
    "\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.ensemble import IsolationForest\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(403, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>Title</th>\n",
       "      <th>EventType</th>\n",
       "      <th>PdfName</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>PaperText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5677</td>\n",
       "      <td>Double or Nothing: Multiplicative Incentive Me...</td>\n",
       "      <td>Poster</td>\n",
       "      <td>5677-double-or-nothing-multiplicative-incentiv...</td>\n",
       "      <td>Crowdsourcing has gained immense popularity in...</td>\n",
       "      <td>Double or Nothing: Multiplicative\\nIncentive M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>5941</td>\n",
       "      <td>Learning with Symmetric Label Noise: The Impor...</td>\n",
       "      <td>Spotlight</td>\n",
       "      <td>5941-learning-with-symmetric-label-noise-the-i...</td>\n",
       "      <td>Convex potential minimisation is the de facto ...</td>\n",
       "      <td>Learning with Symmetric Label Noise: The\\nImpo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>6019</td>\n",
       "      <td>Algorithmic Stability and Uniform Generalization</td>\n",
       "      <td>Poster</td>\n",
       "      <td>6019-algorithmic-stability-and-uniform-general...</td>\n",
       "      <td>One of the central questions in statistical le...</td>\n",
       "      <td>Algorithmic Stability and Uniform Generalizati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>6035</td>\n",
       "      <td>Adaptive Low-Complexity Sequential Inference f...</td>\n",
       "      <td>Poster</td>\n",
       "      <td>6035-adaptive-low-complexity-sequential-infere...</td>\n",
       "      <td>We develop a sequential low-complexity inferen...</td>\n",
       "      <td>Adaptive Low-Complexity Sequential Inference f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5978</td>\n",
       "      <td>Covariance-Controlled Adaptive Langevin Thermo...</td>\n",
       "      <td>Poster</td>\n",
       "      <td>5978-covariance-controlled-adaptive-langevin-t...</td>\n",
       "      <td>Monte Carlo sampling for Bayesian posterior in...</td>\n",
       "      <td>Covariance-Controlled Adaptive Langevin\\nTherm...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Id                                              Title  EventType  \\\n",
       "0  5677  Double or Nothing: Multiplicative Incentive Me...     Poster   \n",
       "1  5941  Learning with Symmetric Label Noise: The Impor...  Spotlight   \n",
       "2  6019   Algorithmic Stability and Uniform Generalization     Poster   \n",
       "3  6035  Adaptive Low-Complexity Sequential Inference f...     Poster   \n",
       "4  5978  Covariance-Controlled Adaptive Langevin Thermo...     Poster   \n",
       "\n",
       "                                             PdfName  \\\n",
       "0  5677-double-or-nothing-multiplicative-incentiv...   \n",
       "1  5941-learning-with-symmetric-label-noise-the-i...   \n",
       "2  6019-algorithmic-stability-and-uniform-general...   \n",
       "3  6035-adaptive-low-complexity-sequential-infere...   \n",
       "4  5978-covariance-controlled-adaptive-langevin-t...   \n",
       "\n",
       "                                            Abstract  \\\n",
       "0  Crowdsourcing has gained immense popularity in...   \n",
       "1  Convex potential minimisation is the de facto ...   \n",
       "2  One of the central questions in statistical le...   \n",
       "3  We develop a sequential low-complexity inferen...   \n",
       "4  Monte Carlo sampling for Bayesian posterior in...   \n",
       "\n",
       "                                           PaperText  \n",
       "0  Double or Nothing: Multiplicative\\nIncentive M...  \n",
       "1  Learning with Symmetric Label Noise: The\\nImpo...  \n",
       "2  Algorithmic Stability and Uniform Generalizati...  \n",
       "3  Adaptive Low-Complexity Sequential Inference f...  \n",
       "4  Covariance-Controlled Adaptive Langevin\\nTherm...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### READ DATA ###\n",
    "\n",
    "df = pd.read_csv(\"Papers.csv\")\n",
    "\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "### DEFINE PREPROCESS FUNCTIONS ###\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def create_sentence(text):\n",
    "    output = []\n",
    "    for s in re.split('\\.\\n|\\.\\r\\n|\\. ', text):\n",
    "        output.extend(re.split('\\n\\n|\\r\\n\\r\\n', s))\n",
    "    return output\n",
    "\n",
    "def clean_sentence(sentence):\n",
    "    sentence = strip_numeric(strip_non_alphanum(sentence)).lower().split()\n",
    "    return [w for w in sentence if w not in stop_words]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "### SELECT A PAPER AND APPLY PREPROCESS ###\n",
    "\n",
    "corpus = create_sentence(df.PaperText[100])\n",
    "cleaned_corpus = list(map(lambda s: clean_sentence(s), corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer()"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### COMPUTE TFIDF WORD SCORES ON PAPER TEXT CORPUS ###\n",
    "\n",
    "tfidf = TfidfVectorizer()\n",
    "tfidf.fit(list(map(' '.join, cleaned_corpus)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1min 26s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "### LOAD GLOVE WORD EMBEDDINGS WITH GENSIM ###\n",
    "\n",
    "emb_dim = 300\n",
    "model = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "### DEFINE FUNCTION FOR SENTENCE EMBEDDINGS ###\n",
    "\n",
    "def calculate_emb(clean_text, original_text):\n",
    "    \n",
    "    tfidf_feat = tfidf.get_feature_names()\n",
    "    tfidf_emb = tfidf.transform(list(map(' '.join, clean_text)))\n",
    "    final_sent_emb = []\n",
    "    final_sent = []\n",
    "    \n",
    "    for row, sent in enumerate(tqdm(clean_text)):\n",
    "        sent_vec = np.zeros(emb_dim)\n",
    "        weight_sum = 0\n",
    "        for word in sent:\n",
    "            try:\n",
    "                vec = model[word]\n",
    "                tfidf_w_score = tfidf_emb[row, tfidf_feat.index(word)]\n",
    "                sent_vec += (vec * tfidf_w_score)\n",
    "                weight_sum += tfidf_w_score\n",
    "            except:\n",
    "                pass\n",
    "\n",
    "        if sum(sent_vec)!=0:\n",
    "            sent_vec /= weight_sum\n",
    "            final_sent.append(original_text[row])\n",
    "            final_sent_emb.append(sent_vec)\n",
    "    \n",
    "    final_sent_emb = np.asarray(final_sent_emb)\n",
    "    final_sent = np.asarray(final_sent)\n",
    "    \n",
    "    return final_sent_emb, final_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████| 312/312 [00:00<00:00, 2405.17it/s]\n"
     ]
    }
   ],
   "source": [
    "### COMPUTE SENTENCE EMBEDDINGS ###\n",
    "\n",
    "final_sent_emb, final_sent = calculate_emb(cleaned_corpus, corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x2bb6510b5c0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAFlCAYAAAApldtwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3df2zc933n+ed75ju/OBxySPGHKFGyLFtWa7tVksqxm9xlHaTeS30LZA9I6+SAba8IYKTbArvA/XHBLtD9N3d/BLiiiwa+btAG2LpeXDet7851VzZqJNdLvFYcM5HkKFZkWRqL5g+RQ87vme98P/fHkAwpDSVSHHL45bweAKHhd76c7+c7HPvFz29zziEiIiL7W6TbBRAREZF7U2CLiIiEgAJbREQkBBTYIiIiIaDAFhERCQEFtoiISAh43S7A3YyMjLgTJ050uxgiIiJ74oc//OG8c2603XP7OrBPnDjB+fPnu10MERGRPWFmH2z2nJrERUREQkCBLSIiEgIKbBERkRBQYIuIiISAAltERCQEFNgiIiIhoMAWEREJAQW2iIhICCiwRUREQkCBLSIiEgL7emlSke14/eI0L53PMVOoMp5J8tzZST732ES3iyUi0hGqYcuB8PrFab7x2hWKdZ8jA0mKdZ9vvHaF1y9Od7toIiIdocCWA+Gl8zkG+zyG+hJEolGG+hIM9nm8dD7X7aKJiHSEAlsOhJlClcHExh6ewYTHTKHapRKJiHSWAlsOhPFMkqWav+HYUs1nPJPsUolERDpLgS0HwnNnJ1kq+yyWawTNJovlGktln+fOTna7aCIiHaFR4nIgrI4Gf+l8jpvLrVHiX/nUCY0SF5EDQ4EtB8bnHptQQIvIgaUmcRERkRBQYIuIiISAAltERCQEFNgiIiIhoMAWEREJAQW2iIhICCiwRUREQkCBLSIiEgIKbBERkRBQYIuIiISAAltERCQEFNgiIiIhoMAWEREJAQW2iIhICCiwRUREQmDHgW1mx8zsH8zsXTO7aGb/qs05ZmZ/bGZXzOzHZvaJnV5XRESkl3gdeA0f+J+dc2+bWQb4oZmdc85dWnfObwKnVr6eBP505V8RERHZgh3XsJ1z0865t1ceF4B3gaO3nfYF4Nuu5QdA1swmdnptERGRXtHRPmwzOwF8HHjztqeOAjfWfZ/jzlAXERGRTXQssM2sH/hr4F8755Zvf7rNj7hNXud5MztvZufn5uY6VTwREZFQ60hgm1mMVlj/R+fcf25zSg44tu77SeBmu9dyzr3gnDvrnDs7OjraieKJiIiEXidGiRvwH4B3nXPf2OS0l4HfWRkt/hSw5Jyb3um1RUREekUnRol/GvgXwE/M7J2VY/8GOA7gnPsm8ArwLHAFKAO/14HrioiI9IwdB7Zz7v+lfR/1+nMc8Ac7vZaIiEiv0kpnIiIiIaDAFhERCQEFtoiISAgosEVEREKgE6PERbru9YvTvHQ+x0yhyngmyXNnJ/ncY1r9VkQODtWwJfRevzjNN167QrHuc2QgSbHu843XrvD6RU31F5GDQ4EtoffS+RyDfR5DfQki0ShDfQkG+zxeOp/rdtFERDpGgS2hN1OoMpjY2LszmPCYKVS7VCIRkc5TYEvojWeSLNX8DceWaj7jmWSXSiQi0nkKbAm9585OslT2WSzXCJpNFss1lso+z52d7HbRREQ6RqPEJfRWR4O/dD7HzeXWKPGvfOqERomLyIGiwJYD4XOPTSigReRAU5O4iIhICCiwRUREQkCBLSIiEgIKbBERkRBQYIuIiISAAltERCQEFNgiIiIhoMAWEREJAQW2iIhICCiwRUREQkCBLSIiEgIKbBERkRBQYIuIiISAAltERCQEtL2m9JTpfIWpXJ6FUp3hdJwzk1kmsqluF0tE5J5Uw5aeMZ2vcO7SDJV6k5H+BJV6k3OXZpjOV7pdNBGRe1JgS8+YyuXJJD0yyRgRMzLJGJmkx1Qu3+2iiYjckwJbesZCqU46sbEXKJ3wWCjVu1QiEZGtU2BLzxhOxynV/A3HSjWf4XS8SyUSEdk6Bbb0jDOTWQpVn0K1QeAchWqDQtXnzGS220UTEbknBbb0jIlsimceHScVjzJfrJGKR3nm0XGNEheRUNC0LumKbk2vmsimFNAiEkqqYcue0/QqEZHtU2DLntP0KhGR7VNgy57T9CoRke1TYMue0/QqEZHtU2DLntP0KhGR7VNgy57T9CoRke3TtC7pCk2vEhHZHtWwRUREQkCBLSIiEgIKbBERkRDoSGCb2bfMbNbMLmzy/NNmtmRm76x8/VEnrisiItIrOjXo7M+BPwG+fZdzvuec+2cdup6IiEhP6UgN2zn3XWChE68lIiIid9rLPuxfN7MpM/s7M3tsD68rIiISens1D/tt4AHnXNHMngX+BjjV7kQzex54HuD48eN7VDwREZH9bU9q2M65ZedcceXxK0DMzEY2OfcF59xZ59zZ0dHRvSieiIjIvrcngW1mh83MVh5/cuW6t/bi2iIiIgdBR5rEzexF4GlgxMxywL8DYgDOuW8CXwR+38x8oAJ8yTnnOnFtERGRXtCRwHbOffkez/8JrWlfIiIich+00pmIiEgIKLBFRERCQIEtIiISAgpsERGREFBgi4iIhIACW0REJAT2amlSEemSqeuLvHJhmpnlKuMDSZ59fIIzx4e6XSwR2SbVsEUOsKnri7zw3fcp1nyODKYo1nxe+O77TF1f7HbRRGSbFNgiB9grF6bJpj2G+hJEIhGG+hJk0x6vXJjudtFEZJvUJC7bNp2vMJXLs1CqM5yOc2Yyy0Q21e1iSRszy1WODG783QwmY9xcqnSpRCJyv1TDlm2Zzlc4d2mGSr3JSH+CSr3JuUszTOcVAPvR+ECSpWpjw7GlaoPxgWSXSiQi90uBLdsylcuTSXpkkjEiZmSSMTJJj6lcvttFkzaefXyCfMlnsVwjCAIWyzXyJZ9nH5/odtFEZJsU2LItC6U66cTGnpR0wmOhVO9SieRuzhwf4vnPPEh/wuPmUoX+hMfzn3lQo8RFQkh92LKpdn3Vw+k4pZpPJhlbO69U8xlOx7tYUrmbM8eHFNAiB4Bq2NLWZn3VEwNJClWfQrVB4ByFaoNC1efMZLbbRRYROdAU2NLWZn3V08tVnnl0nFQ8ynyxRioe5ZlHxzVKXERkl6lJXNpaKNUZ6U9sOJZOeMwXa0xkUwpoEZE9psCWNev7rD+4VaLWCDg23Lf2vPqqRUS6R4EtwC/6rDNJj5H+BNVGk/PXWstXHh1KUar5FKo+T5081OWSioj0JgW2ABv7rAGOD6cB+Gi5QiIWYTgd56mTh9QULiLSJQpsAdr3WU8O9ZGMRfkfn3ygS6USEZFVGiUuAGvzq9dTn7WIyP6hwBYAzkxmNb9aRGQfU5O4ADCRTfHMo+NM5fLMF2v33WetnbxERHaHatiyZiKb4vOPT/DZ02MA/MPlWV69ML3lnbi0k5eIyO5RYMsGOwld7eQlIrJ7FNiywU5CVzt5iYjsHgW2bLCT0NVIcxGR3aPAlg12EroaaS4isnsU2LLBTkJ3daS5dvISEek8TeuSDXY6vUs7eYmI7A4FttxBoSsisv+oSVxERCQEFNgiIiIhoMAWEREJAQW2iIhICCiwRUREQkCBLSIiEgIKbBERkRBQYIuIiISAAltERCQEFNgiIiIhoMAWEREJAQW2iIhICHQksM3sW2Y2a2YXNnnezOyPzeyKmf3YzD7RieuKiIj0ik7VsP8c+Pxdnv9N4NTK1/PAn3bouiIiIj2hI9trOue+a2Yn7nLKF4BvO+cc8AMzy5rZhHNuuhPX329evzjNS+dzzBSqjGeSPHd2ks89NtHtYomISIjtVR/2UeDGuu9zK8cOnNcvTvON165QrPscGUhSrPt847UrvH7xQP5tIiIie2SvAtvaHHNtTzR73szOm9n5ubm5XS5W5710Psdgn8dQX4JINMpQX4LBPo+Xzue6XTQREQmxvQrsHHBs3feTwM12JzrnXnDOnXXOnR0dHd2TwnXSTKHKYGJjT8NgwmOmUO1SiURE5CDYq8B+GfidldHiTwFLB7X/ejyTZKnmbzi2VPMZzyS7VCIRETkIOjWt60Xg+8BpM8uZ2VfM7Ktm9tWVU14BrgJXgP8D+JeduO5+9NzZSZbKPovlGkGzyWK5xlLZ57mzk90umoiIhFinRol/+R7PO+APOnGt/W51NPhL53PcXG6NEv/Kp05olLiIiOxIRwJbNvrcYxMKaBER6SgtTSoiIhICCmwREZEQUGCLiIiEgAJbREQkBBTYIiIiIaDAFhERCQEFtoiISAgosEVEREJAgS0iIhICCmwREZEQUGCLiIiEgAJbREQkBBTYIiIiIaDAFhERCQEFtoiISAgosEVEREJAgS0iIhICCmwREZEQUGCLiIiEgAJbREQkBBTYIiIiIaDAFhERCQEFtoiISAgosEVERELA63YBZO9M5ytM5fIslOoMp+OcmcwykU11u1giIrIFqmH3iOl8hXOXZqjUm4z0J6jUm5y7NMN0vtLtoomIyBYosHvEVC5PJumRScaImJFJxsgkPaZy+W4XTUREtkBN4gfEvZq7F0p1RvoTG34mnfCYL9b2uqgiInIfVMM+ALbS3D2cjlOq+Rt+rlTzGU7H97q4IiJyHxTYB8BWmrvPTGYpVH0K1QaBcxSqDQpVnzOT2S6WXEREtkpN4gfA3Zq71zeVx6JGteFTaTQZTsd56uQhjRIXEQkJBfYBsNrcnUnG1o6Vaj6G49ylGTJJj5H+BKWaT6Hq88yjYwpqEZGQUWCHyGYDy85MZjl3aQZo1axXgzkWNTLJ6FqQr/47lcsrsEVEQkZ92CFxt4FlE9kUzzw6TioeZb5YIxWP8syj4zhaAb5eOuGxUKp35yZEROS+qYYdEusHlsGdteXVr/U2ayrXyHARkfBRDTskFkr1bdeWNTJcROTgUGCHxP3Mo96sqVz91yIi4aMm8ZDYbGDZUycP3fXn2jWVi4hI+KiGHRKqLYuI9DbVsENEtWURkd6lGraIiEgIKLBFRERCQIEtIiISAh0JbDP7vJldNrMrZva1Ns8/bWZLZvbOytcfdeK6IiIivWLHg87MLAr8e+AZIAe8ZWYvO+cu3Xbq95xz/2yn1xMREelFnahhfxK44py76pyrA38FfKEDrysiIiIrOhHYR4Eb677PrRy73a+b2ZSZ/Z2ZPdaB64qIiPSMTszDtjbH3G3fvw084JwrmtmzwN8Ap9q+mNnzwPMAx48f70DxwmmzrTRFRKQ3daKGnQOOrft+Eri5/gTn3LJzrrjy+BUgZmYj7V7MOfeCc+6sc+7s6OhoB4oXPnfbSnMrP/vqhWn+8s0PePXC9JZ+RkRE9r9OBPZbwCkze9DM4sCXgJfXn2Bmh83MVh5/cuW6tzpw7QNp/VaaETMyyRiZpMdULn/Xn9tJ0IuIyP624yZx55xvZn8I/D0QBb7lnLtoZl9def6bwBeB3zczH6gAX3LO3d5sLisWSnVG+hMbjqUTHvPF2l1/7l57ZouISHh1ZC3xlWbuV2479s11j/8E+JNOXKsXrG6luRq4cO+tNOH+g15ERPY/rXS2D52ZzFKo+hSqDQLnKFQbFKo+Zyazd/25+9kzW0REwkGBvQ/d71aa9xv0IiKy/2l7zX3qfrbSXA36qVye+WKN4XScp04eUv+1iMgBoMDugt2cY609s0VEDiY1ie8xTb0SEZH7ocDeY/c7x1pERHqbAnuPLZTqpBMbeyLSCY+FUr1LJRIRkTBQH/Yems5X+OBWiR9dzzPan+DESB/D6YSmXomIyD0psHfJ7QPLJgaSTOWWODyQYrnis1St86PrdU6N9+NFIjx18lC3iywiIvuYmsR3QbuBZS++dYNm4Dg23MfHjw8xmIrhBwEzy9UtzbEWEZHephr2Lmi3pnczcMwWKhwb7mM4HWc4PUzgHPPFmsJaRETuSYG9C9qt6X0oHb9jTe+t9F1rX2wREQEF9q5ot3nH2ECCfKVOodognfAo1XwKVf+ufderTeuZpMdIf2tw2rlLM5yZHGR6uaoQFxHpIerD3gXt1vT2IhG+/MTxba0P3m7Oth8EvPjWdS28IiLSY1TD3gW3r+ltQCwKF6eXGU7H+ezpsS3ViNs1rc8u12gGTntei4j0GNWwd8lENsXnH5/gs6fHaDQdyZi37Rpxu+0yb5XqjPRv7PfWwisiIgefatgddvsgscVS7Y4R47C1GvGZySznLs0ArPV7RyPGWGbjz2nhFRGRg0817A5qN//6+1cXqPnNDedttUbcbl/sLz9xjGjEtOe1iEiPUQ27g9rNvz6UjnP5owIjDyfXzru9Rny3qVvttsscG0hqz2sRkR6jwO6gdoPETh8e4B+vzG86nWuzqVt3G0GuPa9FRHqPmsQ7qN0gsYQX4VMPHWo7nWs6X+HP/7/3+cmHed6bLZAvN7TdpoiItKUadge1GyRWqPpta8urNeuFUoPR/gQ1P+CdG3k+dixLti92x6poIiLS21TD7qB2g8Q2a9pe7e9uhbUjFfNIxSNcu1XUqG8REblDz9awd2uN7q32L6/2d58Y6eOdG0tAq/l8rlDj8EBK222KiMgGPRnY9zPQq9MM+P7P5/EDR63R4MN8mULVZyQd58zkoAaViYjIBj0Z2O2mX60e34ugnM5XmC/UWK76RCPwYb6KHzgmh1KcHEnz4lvX+cefz3NytF8be4iICNCjfdgLpTrpxMa/VfZyec+pXJ7J4T6efHCYpbJPJBJhMBUjFoHZQp1oxFiu+NrYQ0RE1vRkYLebfrWXA71W/2AYTicYG0jw8WNZfunwAHPFBql4hGwqTqnua4qXiIis6cnAbrf95V4u77n+D4ZMIka1EVD1m5hB0otSbQRkEq1mem3sISIi0KN92Ldvf7nXy3uun699/FCKt64t4hwcH0qRr9Rxzjh9uB/Y3jKmIiJycPVkYEN3l/dc/wdDpdHk7ANZwMiXG3ywUOaR8TTZvvhazX8ny5iKiMjB0LOB3W23/8GwWnMGmFmuUm00OTnav6Hm3+3R7SIi0j092Ye936zWnKfzVZardRZKda4vlJkYSG4I4m6PbhcRke5RYO8DU7k8zcDx3myRetMxlkkSjRgvvnVjw5Subo9uFxGR7lFg7wMLpTqzhQqpeIRUzMPMyKbiNAO3YUpXt0e3i4hI9yiw94HhdJz5Yp2kF107Vm0EHErHNzR3b2dzEREROVg06GwfODOZ5Y3Ls+QrdbKpONVGQKXR5OhQ+o7m7m6ObhcRke5RDXsfmMim+PITx2kGMFuoEYsaD4+l8SIRNXeLiAigGvau28pCJ9P5CtPLVY4Pp1iqNBhMxTiSTWlRFBERWaPA3kV3W+gE4I3LM7x5dZFcvszJkTSfeGCIicHU2kAyhbWIiKxSYN+HrS4PutlCJ29cnmG+UOfaQplCtUFfPMqNxQp1P+BTD4+QSXq8cXmGoXRCS5CKiAigPuxtW601V+pNRvoTa1tgTl1f5NUL0/zlmx/w6oVppvOVTRc6uXBzmYVya4BZE0d/IkZ/Ikax2uTafJma3+T7VxfuuIa22RQR6V0K7G1aX2uOmJFJxmgGjhffunFHwBqu7UInzkG+XOfDfJn55RofLpbxm00arkmh1uDyRwUOpeMbrqFtNkVEepuaxLdpoVRnpD+x4dhsoUIzcHc0fVcbTQrVVmCnEx65xTI/myny4WKJK7Ml4l6ETNKjWPMpNZqMpeN4EWO+WOfTD49suEY64TFfrO3BHYqIyH7UkcA2s88D/zsQBf7MOff12563leefBcrA/+Sce7sT195rq8uDroYywHyxTsKL8vb1BQpVn0zSYyAZ48ZimclsHx/myxiQrzSYGEzy89kCXsSoNJrEoxEc0PCbFOtNHhnv55FxI+FtbPzQEqQiIr1tx4FtZlHg3wPPADngLTN72Tl3ad1pvwmcWvl6EvjTlX/3pbsNKlu/l3U64VGq+dT8gELFJxGLMJCMMV+s8sZP58imYxgQi0a4VarzsWNZ5os1Yl6EU2MZbi5XKNaapONRjg2leOrkCF9+8sRaP/n6a6zfZlNERHpPJ/qwPwlccc5ddc7Vgb8CvnDbOV8Avu1afgBkzWyiA9fuuM0Gla0O+Gq3POgvH+4nnfTAGQC5hQqFmk8EGEzFiUSMGwtlrs4VKVR9BpIxGk2HFzGaQUA0YhRqPtm+2KbX0BKkIiK9rRNN4keBG+u+z3Fn7bndOUeB6Q5cv6O2suf0+uVBp/MVvvfeHIELyC2WSSciLJYbHB9O4nkRzIxUzGOoL8bP50p8/HiWfLnB9cVlKvUmqViEZhBwq+hzdb7EdL6iYBYRkTt0ooZtbY65+zindaLZ82Z23szOz83N7bhw27WdPaen8xX+z7dz3MxXmVmq0Ww6ggDinrFYqjO3XOPqbJFitcFIJkmjGTDan2SxXCNirRq2F40SsQi/cnQAvxkwlcvfs5YvIiK9pxOBnQOOrft+Erh5H+cA4Jx7wTl31jl3dnR0tAPF257b95xeKNX4/s/nufDh0tr86lVvXJ7l/bkS4wNJvGiEhgv4YKHMXKFKsdZkqC9OvRlweaZAodrgiQeG+Gi5wq1inXItIB41krEI4wMJhtMJGs2AhVK97dQxTesSEeltnQjst4BTZvagmcWBLwEv33bOy8DvWMtTwJJzbt81h8PGPafni1XefH+BpYrPr05m76jp/uTDJbJ9rWVHT470k45HKdUaJGIevzQxQCIWodEMiEWNht+kLxnjlw4P8GsPZOlPxoh6UcYyCWLRKO/NFqk1mgyvbKm51Vq+iIj0hh33YTvnfDP7Q+DvaU3r+pZz7qKZfXXl+W8Cr9Ca0nWF1rSu39vpdXfL6oCvqVye8x8sMJD0+OWJwQ1TqlaXDb0yV6A/FuPYoT4yyRj9yQz5SgMvYvzGLx/m2q3WILP+RJTppSqT2dRKrTnKYMpjuepzq1hnbLBVu641HWcms0zl8ndMHdO0LhGR3taRedjOuVdohfL6Y99c99gBf9CJa+2F1UFl78+VWK7W+cmHrSbqE4f6CVzA968u8NnTY5wey/DuRwV+9lGRU+NpYtEoQeAYH0wynI4znB4GoFBtsFz112rNDsevHM3ys5llrt0qU2k2OTKQZHLdYDZN6xIRkfWslaX709mzZ9358+f3/LrT+QpvXJ7lr9/O0RePcuJQmpgXoVIP8JsB6USUTz88ykKpzmvvznDjVpGGcxzL9jGY9AistfDJSH+csUyKYt3no3yZpYpPseZzbb5ExW/iNx0j/XGePHmI5UqDuUKd/+bhER4cTTMxkGR6uarNP0REeoiZ/dA5d7bdc1qa9DarI7Tfny9ycjTNjYUK782UOJpNMFOocX2hzGdOjfyiP9k5AgdLJZ9ms4QXMf7JI6N40SjzxTofLlYYH0wxmkny49ws+XKdUq1B3Xc4g3KtydvXFohEIpwcTbNcrVOpJ5nKLWnutYiIrOmZwN7ulph+4BjtT9IX83hvtsiPcks8MJwi2xfj2q0yP50pEjVYLNfJVxs4HHPFGn7T8X9N3eS3nzjGf/fYYf7xyhx+M6DeNAaSUZYrkIh5BK7VRB7gWK76HMv2MZCMcXG6QLHWxIsYb1yGLz95Yu/fLBER2Xd6IrBXa82ZZGtEd6nmc+7STNsa7OrmHpmkR9Vv0p+MMZCM4kgw2p+kUPXxm5CKRfjp9DKVemulsnrTxw/AOVgoN3jxv97gymyJUr3JoXQMP4gR86L0p2IkohHmi3WG+mL4gaPuB5QbPm9evQUG6ViETCrO968u8PRp1bJFRKRHttfczrzm1XnYJw71U6kHVBo+S1WfZDTKzaUqD43288jhfsyMfKVBsd4kX/Wp+a2wdg6aDoo1nx9dX2BuucIHt8qAIxoxPDPqfsBgKkbVD3DOYRG4uVzFd3Ak24cfwI2FMkkvornXIiIC9Ehgb2de8+o87FjU+NXJQYLAUaj6VBoNitUGS+UGpVqD+UIVc+CtrOHWdOAH4LtfBPdCucGtcp3cQoV3p5eJ4HC0du0q1xuU6z6Vho/faBKLGBMDSVKxKKsLwyW8iOZei4gI0CNN4u22xNxsXvP6ediVRpNHxjNUG02uzpXIV6rMFapU/ICG39plq9Z0NBrBhtdwQLO5Mqis2qTqByzl6gz1JRjq84hHIRqJcCgRIRX3WKw0GE22+rPLDZ8+z+PUeD+Vemshla32v4uIyMHVE4HdbkvMQtXn5EiaVy9M3xGEq2H4xuVZvvOjHDfzFQwoNZrU6k2qDUfAnYuhr//eByIOan6AGQQOEp5R8x2nJwb5+LEs125VSMWivD9foFD1aTYCjg/1cag/Qb5SJ7pS695q/7uIiBxcPdEk3m67yjOTg0zlltpusDGdr/Dn//g+fzt1kyuzRYrVJkvVBgQBkQg02WTnktsEQMNBY2UwWqneZLnaoNYI+OEHeVKxKKl4lMnhNJgxOZxisdRgtlCjGcCXnzjO9HJV64qLiEhv1LBh45aYAK9emN50G83FUp2pD/MkvOjaNmPVRoBb6ae+X6VaE8Mxt1xp7dIFVJpNomYcH0oxNpBgeqnK5355bK22f/HNDxjpT2x4nXTCY75Yu/+CiIhI6PRMYN9udfrWejU/4K1rC1yZKzC3XCPuRSjVm9T9Vn16B1ndGozmByTjEa4vVhjqi5FJxojHjFKtSSYZYyyT4syxIT7/+MTaz22n/11ERA6ung3s1SBsNB3XbhV5f7bEz28VySRiLFfq3CpWaTSNZrDzsF7lRcE5R63hqDWafLhU5nAmwSNjA/gu4PLMMp9//PCGn9ms/13riouIdMeJr/0/dxy79vX/ftev2xN92O2cmcySy1f4wdVbzBeqXJ0v4vuOqEGtEVBpOJxrNYPfT1hH7c5jjSatAWuBIxqNMNQXZ6HsM7NcJpOIcXy4746BZO363zXgTESkO9qF9d2Od1LP1rAnsilG0jEWih6Xbi5TqDVJesZsocZcsUoQAJFWU/b9CFzrryG38pXyWrX1RgB1B3PFOmOZFIMpj1LdMTaQ4MgmIXx7/7uIiPSenq1hAziM04cHCICBRKybYZwAAA41SURBVIRaM6DWCGgEEIv+InTvhwHrup2pNx3NdVX1uu8o1X3mizVuLJR45SfTLJbqTOcrO7gjERE5qHo6sIfTcS5/tMxA0sOs9VY0goBE1HBmGBC9z3coHoWhviQGxCPgRWxt7nYM8CJQqTdZKjdoBgHHhvtIxqJrU8tERETW6+nAPjOZ5VapzqG+OI3A0Rdv7dKVikfxIhESHkQjbTqj78FoNaUbjngMIhGIexGikdZSptEopOJRan6TmBcBM8q1Jo1moDnWIiLSVk8H9kQ2xa+fHG6t8G2Ocs0nFYuSTcX41ckBsukEw/1xMokIicjW36yotdYBH+iLMZSKE7UI6YTHUF8ML9JaRCURjVBrNEl6UfpXnnvnxhI1v6n1w0VE9qnNRoPvxSjxnh10turRiUFeuzTLcCrOQlAncFCqBzyUjHFkMEXND0hEoyyVa5TrAfWmu+eo8YRnRAyageOhsX5mlqqU600qviOViPHAUJLZYp16ELBQrnM0m2C54jOQ8rj8UYEnNWVLRGTf2otwbqfnA/vS9DLxWISMxYhEjFLNJxo18qU6Q+k40/kqQ30xKnUf30HTNVsjyO+i3nQcSkQZ6osTIcJENsVnT49zeWYZz+D9W2UG/IC+WJRG4PADWK7UWSjX6E94PP+Zh/bm5kVEJDR6PrB/8uESE4NJ+uK/GNI9W6hwda7Ebz4+Qc0PuPzRMnOFCkHNgYMobNj8w9Y9TkYhlfAYziQZSSeYL9U4PJgkGYtwZnKQCzeXeWQ8w9VoBBx4kQjzxSqL1QbDfXEODyY1hUtERO7Q84FtBriNA8vmC3Vi0ZWNNoDlwSSOCH1xj8GUUar7FGsba9rxCDwynqFQ86n5Aam4R6nu8+mHRvjMI6NM5ZbIJD0MwwwWS3WS8SiDyShHsimWqg0mh/rumIutrTVFRAR6fNAZwONHBliqNqjUmzjnqNSbLJYbPDSaXjtndWetbF+M4f44qbhHNhUjGTOSsQixaGv617WFMpWGzwOH+hhLx/GdIxo1XrkwTTNwZJIxBpIxjAijmQQxay1XulT1ySRinBrv5+Ro/9p1p/MVzl2aabujmIiI9JaeD+ynT49zYriPwAUsVRoELmByKMVDowNr5+QrdZKxCKlYlOPD/Tx6eJBULEoQQH/SI9sXw/Nau2+lYh5jmSQ3l2pMDCR5eCzDQqnBz2YKLJTqnBjpo9Jocqg/Tr3pGE7HeWC4jydPDuNFIpyZzK5ddyqX19aaIiICqEmciWyKL549tqHZeWIgyVRuiUK1QTrh0ReLtqZ8xT3qfkAiFqHmByTjUR4Zy5Av18lXGhhGvRnw/q0SSS9CwosSMWO0P8FStc61W0U+cXyYjx0b5N3pZY4OpehPeAymYhzJpu5o7m63o5i21hQR6U09H9jQfq3usYEkU7k8P58tMpjyuLlUJRkE1JpNlqo+gXOM9scYSMXIV+tkUzEK1QaNAKqNJh+bHGR1M+1sn8ePc3nKjVa/99hAggdH+nn+0XGAtT8WVmvOq2XR1poiIrKq55vENzOxUuNNJzz+yelx/oePH6E/4bFYqvPIWIbf+rWjHOpPUqo1iUdab2MyHuOhQ2lOjvYTAJlEjIVSjYs3l6k3m1TqPlO5RX50fZEzk4MAd+2jPjOZpVD1KVQbBM5RqDYoVP0NzeYiItIbVMO+i/V9yJlkjOF0gnenl6n5ARFac7XnSgW8SIRYBIbSCQ4NJDg5kua9mRIPjfbz4xt5buYreNEInzxxCC8aIV+pc2l6ienl6trrA2v/TuXya7X+Zx4dZyqXZ75YYzgd56mThzRKXESkBymw72J9H/LVuSLfe2+OpnOUqg0aTUjEozzc18+tcoNyvcmRbJJfOpzh5Gg/nz09xvRylVcvTDOQjHFsOE1/svV2Oxfjws1lHjsyeM8+am2tKSIioMC+q9U+5EbT8b335omYkfCi5BYqxL0Ih9Ip+lNRnnhwhMWVVcr+5WdPrf38GeC7P5sjGWvN4V5jDufURy0iIlunPuy7WO1Dfnd6iaYLiEUj1JsBXsToi0co1RtU6q3VUwaTMWaWq3e8xq8cHSRf9qk0/NY874ZPvuzzK0cH1UctIiJbpsC+i9U+5FZIt5YSPTmSZrAvTjOAcsMnFW+9hUvVBuMDyTte4+nTYzw4miYIHEuVOkHgeHA0zdOnx9ZePxWPMl+skYpHeebRcTWBi4jIHdQkfg8T2RT/7alRbuYrXJktEY1EeGikjzevLRKPRhhJx1ks18iXfH771461/fkvfmJy0+VF1UctIiJbocDegjOTWWaXazw8lmZ2uYZFIpwe6yeT8ijUfMYHkvz2rx3jzPGhtj+vUBYRkZ1SYG/B+ulVyViUjx3PahMOERHZUwrsLVItWUREukmBfRttZykiIvuRRomvo+0sRURkv1INe531S5HCnUuFrrdZTXz98dbeHw6HqbYuIiI7ohr2OgulOunExr9h0gmPhVJ9w7HNauJT1xfXjkfMeOvaIuc/yBMxVFsXEZEdUWCvs7pU6HrtlgpdXxOPmK1sDuLxyoXptePXF0oMpWNkU3Gu36qsnbO6haaIiMh2KLDX2epSoZvVxGeWq2vHC1WfpBclGYtQqDXWzrm9ti4iIrIVCux1trpU6GY18fGB5NrxTNKj6jepNgIyidjaOdrYQ0RE7ocGnd1mK/Otz0xmOXdpBmjVmks1n0LV59nHJ5jKLQFwfDjNf31/ATN44sTQWm39qZOHdv0eRETk4NlRYJvZMPAScAK4Bvy2c26xzXnXgALQBHzn3NmdXLfb1q98Nl+sMZyO89TJQ0xkU4wNJJnK5ak0mjxxYghwBA5S8ejaOSIiItu10xr214DXnXNfN7OvrXz/v2xy7medc/M7vN6+sVlNXCuiiYjIbthpH/YXgL9YefwXwD/f4euJiIhIGzsN7HHn3DTAyr9jm5zngP9iZj80s+d3eE0REZGec88mcTN7DTjc5ql/u43rfNo5d9PMxoBzZvZT59x3N7ne88DzAMePH9/GJURERA6uewa2c+43NnvOzGbMbMI5N21mE8DsJq9xc+XfWTP7DvBJoG1gO+deAF4AOHv2rLv3LYiIiBx8O20Sfxn43ZXHvwv87e0nmFnazDKrj4F/ClzY4XVFRER6yk5HiX8d+E9m9hXgOvBbAGZ2BPgz59yzwDjwHTNbvd5fOude3eF1Q0/beIqIyHbsKLCdc7eAz7U5fhN4duXxVeDMTq5z0KxuHpJJeoz0JyjVfM5dmmm7qpqIiAhoadKu2GzzEG0MIiIim1Fgd8FWt/EUERFZpcDugq1u4ykiIrJKgd0FW93GU0REZJUCuwu2uo2niIjIKm2v2SXaJERERLZDNWwREZEQUGCLiIiEgAJbREQkBBTYIiIiIaDAFhERCQEFtoiISAgosEVEREJAgS0iIhICCmwREZEQUGCLiIiEgAJbREQkBMw51+0ybMrM5oAPul2OLRgB5rtdiC7q9fsHvQe9fv+g96DX7x868x484JwbbffEvg7ssDCz8865s90uR7f0+v2D3oNev3/Qe9Dr9w+7/x6oSVxERCQEFNgiIiIhoMDujBe6XYAu6/X7B70HvX7/oPeg1+8fdvk9UB+2iIhICKiGLSIiEgIK7PtgZsNmds7M3lv5d2iT866Z2U/M7B0zO7/X5ew0M/u8mV02sytm9rU2z5uZ/fHK8z82s090o5y7ZQv3/7SZLa38vt8xsz/qRjl3i5l9y8xmzezCJs8f6N8/bOk9OOifgWNm9g9m9q6ZXTSzf9XmnAP7Odji/e/eZ8A5p69tfgH/G/C1lcdfA/7XTc67Box0u7wduuco8HPgJBAHpoBHbzvnWeDvAAOeAt7sdrn3+P6fBv7vbpd1F9+DzwCfAC5s8vyB/f1v4z046J+BCeATK48zwM967P8DW7n/XfsMqIZ9f74A/MXK478A/nkXy7JXPglccc5ddc7Vgb+i9T6s9wXg267lB0DWzCb2uqC7ZCv3f6A5574LLNzllIP8+we29B4caM65aefc2yuPC8C7wNHbTjuwn4Mt3v+uUWDfn3Hn3DS0foHA2CbnOeC/mNkPzez5PSvd7jgK3Fj3fY47P6hbOSestnpvv25mU2b2d2b22N4Ubd84yL//7eiJz4CZnQA+Drx521M98Tm4y/3DLn0GvE690EFjZq8Bh9s89W+38TKfds7dNLMx4JyZ/XTlL/QwsjbHbp9isJVzwmor9/Y2rWUFi2b2LPA3wKldL9n+cZB//1vVE58BM+sH/hr418655dufbvMjB+pzcI/737XPgGrYm3DO/YZz7vE2X38LzKw28az8O7vJa9xc+XcW+A6tZtWwygHH1n0/Cdy8j3PC6p735pxbds4VVx6/AsTMbGTvith1B/n3vyW98BkwsxitsPqPzrn/3OaUA/05uNf97+ZnQIF9f14Gfnfl8e8Cf3v7CWaWNrPM6mPgnwJtR5aGxFvAKTN70MziwJdovQ/rvQz8zsoo0aeApdWugwPgnvdvZofNzFYef5LWf1+39ryk3XOQf/9bctA/Ayv39h+Ad51z39jktAP7OdjK/e/mZ0BN4vfn68B/MrOvANeB3wIwsyPAnznnngXGge+s/N484C+dc692qbw75pzzzewPgb+nNWL6W865i2b21ZXnvwm8QmuE6BWgDPxet8rbaVu8/y8Cv29mPlABvuRWho0eBGb2Iq0RsCNmlgP+HRCDg//7X7WF9+BAfwaATwP/AviJmb2zcuzfAMehJz4HW7n/XfsMaKUzERGREFCTuIiISAgosEVEREJAgS0iIhICCmwREZEQUGCLiIiEgAJbREQkBBTYIiIiIaDAFhERCYH/HxoFOqsMoCiOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### 2D REPRESENTATION OF SENTENCES ###\n",
    "\n",
    "pca = PCA(n_components=2)\n",
    "X = pca.fit_transform(final_sent_emb)\n",
    "\n",
    "plt.figure(figsize=(8,6))\n",
    "plt.scatter(X.T[0], X.T[1], alpha=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x2bb651c1c50>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAFlCAYAAAApldtwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5wcdf3H8ddn97bd5dJ7IyABBH5Ew0lABEIJVYoKAkqAiESkyQ9UIiD8BFRQQIwgvSsiSItIlxakmEuAUEIgCWkE0tuV7d/fH7sJubu9yyW35eb2/Xw87pHdme/MfGZv4X3znZnvmHMOERER6dx8pS5ARERENk+BLSIi4gEKbBEREQ9QYIuIiHiAAltERMQDFNgiIiIeUFHqAtrSt29fN2LEiFKXISIiUhTTp09f4Zzrl2tepw7sESNGUFtbW+oyREREisLMFrQ2T13iIiIiHqDAFhER8QAFtoiIiAcosEVERDxAgS0iIuIBCmwREREPUGCLiIh4gAJbRETEAxTYIiIiHqDAli6lsTHGkrmfkUqlSl2KiEhedeqhSUXaq7Exxg92OJcVn67aOG3PI3fniscnlbAqEZH80RG2dAknb3tmk7AGeOOf0/nDj24uUUUiIvmlwBbPW/7pStYsW5dz3tN3vljkakRECkOBLZ73wWuzW52XTqWLWImISOEosMXzvnLg/7Q6zx/wF7ESEZHCUWCL5/XoXc3QHQfnnPe9i75d5GpERApDgS1dwu3vXccue++48b35jBMmHcPJl323hFWJiOSPbuuSLsHv93P91CtLXYaISMHoCFtERMQDFNgiIiIeoMAWERHxAAW2iIiIByiwRUREPECBLSIi4gEKbBEREQ9QYIuIiHiAAltERMQDFNgiIiIeoMAWERHxAAW2iIiIByiwRUREPECBLSIi4gEKbBEREQ/ocGCb2TAze9HMZpnZ+2b2kxxtzMwmm9kcM5tpZqM7ul0REZFyUpGHdSSBC5xzM8ysGphuZs855z7YpM1hwMjszxjgpuy/IiIi0g4dPsJ2zn3mnJuRfb0emAUMadbsaOBel/EG0NPMBnV02yIiIuUir+ewzWwE8FXgzWazhgCLNnm/mJahLiIiIq3IW2CbWTfgYeA859y65rNzLOJaWc9EM6s1s9rly5fnqzwRERFPy0tgm1mATFj/1Tn3SI4mi4Fhm7wfCizJtS7n3K3OuRrnXE2/fv3yUZ6IiIjn5eMqcQPuAGY5565rpdkU4OTs1eJ7Amudc591dNsiIiLlIh9Xie8NjAfeNbO3s9MuAoYDOOduBp4EDgfmAA3AhDxsV0REpGx0OLCdc6+S+xz1pm0ccFZHtyUiIlKuNNKZiIiIByiwRUREPECBLSIi4gEKbOky5r07n9em/JdUKlXqUkRE8i4fV4mLlNTs6XM4b+9fkownN047aPy+XHjPOSWsSkQkv3SELZ53zpiLmoQ1wPP3vcKjf3qqRBWJiOSfAls87fEbn8Klc45yyz2XPlDkakRECkeBLZ724X/ntDqvsS5axEpERApLgS2edvAp+7U6r9+wPkWsRESksBTY4mlfPWA3qvt0yznvovt/UuRqREQKR4EtnvfQ57ez05iRGwfIrepRya+fmMTOe+5Y2sJERPJIt3WJ5/n9fv70+m9KXYaISEHpCFtERMQDFNgiIiIeoMAWERHxAAW2iIiIByiwRUREPECBLSIi4gEKbBEREQ9QYIuIiHiAAltERMQDFNgiIiIeoMAWERHxAAW2iIiIByiwRUREPECBLWXj8/nL+OVRV3F45ESO6jGeyWfeRmNdY6nLEhFpFz1eU8pC3Zp6zt7jF6xftZ502pGIJXn6rheYO3MB10+9AjMrdYkiIm3SEbaUhWfveYloQ5R02m2cloglmffOfGZPm1PCykRE2keBLWXho+lziTXEc86b//7iIlcjIrLlFNhSFr40agShSLDlDDOG7zS4+AWJiGwhBbaUhUMm7E8wEmxyrjoQrGD4ToP58p47lLAyEZH2UWBLWejeu5rJr/+GUfvvgvmMQKiCsSfsze+eu1QXnImIJ+gqcSmJhR9+ykPXTOGTdxew49e259gLjmTQtgMKus2hIwfx++cvI51OY2YKahHxFAW2FN37r81m0sFXEI8lSKfSzHlrPs/d9zLXT72S7XbbpuDb9/nUsSQi3qP/c0nRTT7rNqINMdKpNACpZIrG9VFuOv/u0hYmItKJKbClqJKJJJ/MXJhz3vv/mV3kakREvEOBLUXlr/ATDAdyzqvqESlyNSIi3qHAlqIyMw774YEEm90THaoMcvTZh5WoKhGRzk+BLUV3+u/GM+aI0QTDAap6VBIIBRh7/N6cOOlbpS5NRKTT0lXiUnTBUIBLH7yA5YtXsmTu5wzbcTC9B/YqdVkiIp2aAltKpt/QPvQb2qfUZYiIeIK6xEVERDxAgS0iIuIBCmwREREPyEtgm9mdZrbMzN5rZf5YM1trZm9nfy7Nx3ZFRETKRb4uOrsbuAG4t402U51z38zT9kRERMpKXo6wnXOvAKvysS4RERFpqZjnsPcys3fM7Ckz26WI2xUREfG8Yt2HPQPYxjlXZ2aHA48BI3M1NLOJwESA4cOHF6k8ERGRzq0oR9jOuXXOubrs6yeBgJn1baXtrc65GudcTb9+/YpRnoiISKdXlMA2s4FmZtnXe2S3u7IY2xYREekK8tIlbmZ/A8YCfc1sMXAZEABwzt0MHAv82MySQCNwgnPO5WPbIiIi5SAvge2cO3Ez828gc9uXiIiIbAWNdCYiIuIBCmwREREPUGCLiIh4gAJbRETEAxTYIiIiHlCskc5EpESWLVzO/b95hMa6KN8842D+5xtfLnVJIrIVFNgiXdjff/cYt0/668b3L9z/Kv+z75e57qXLS1iViGwNdYmLdFFrVqxrEtYbvPvKLJ645bkSVCQiHaHAli1Wv66B2dPmsPKz1aUuRdrwyHVPtDrv4etbnycinZO6xKXdnHPc839/56Fr/klFwE8ynuRrh36VSX85l3BlqNTlSTOxaKLVecl4soiViEg+6Ahb2u25e1/m4WufIN4Yp2FdI/FogmlPv8Ufz7i11KVJDsecc2ir8w4+df8iViIi+aDAlnZ78PePE22INZkWjyZ4+aHXaayPlqgqac2gbQdw6A9aBnP/4X35/sXfLkFFItIR6hKXVqXTaRZ9+CnBcJBB2w1gzfJ1OduZQcO6RiJV4SJXKJtzwe1nst/xe3P/lf+gsS7GQeP35VvnHo7Pp7/VRbxGgS05vfXCu1x10mQa6qK4VJrB2w9k5O7bMf3Zd3Dppk9G7dazil4DepSoUtmcmnGjqBk3qtRliEgHKbClhWWLVnDp0VcTrf+i+3v++4tYvXQNkW5hYg1xUskUZhCMBDnnxh/qiE1EpMAU2NLCU3f8m1Qi1WSaSztijQnO+uMEPnzzY96dOoshIwdx/IXHsPOeO5SoUhGR8qHAFiBzy9ab/5rBv257jo9r55HIcduPS6dxacdPbppYggpFRMqbAlsAuPn8u3ny9n836QZvLp127Pz1HYtYlYiIbKDAFpbM/ZwnbnmOeBsDbYSrQuz9rTEM32lIESsTEZENFNjC2y++j8+f+6Kxbr2qGLTdAI768SEcfOrY4hYmIiIbKbCF6l5VWI6rvCsCfo468xAmXHFiCaoSEZFN6V4cYcwRo/HnOML2V/g5dMIBJahIRESaU2ALwXCQq579Jb0G9KCyOkJl9wiRbmEuvPccBm03YIvXt27Vej6d8xmpVGrzjUVEpF3UJS4A7FjzJf62+BY+fHMO0fooqVQal0rTsL6RyupIu9ZRv7aeq0++gdpn38Zf4ScYDnLODacx9vi9C1y9iEjXp8CWjfx+P/4KH7/5/h83Pn4xlUhxzo0/5JB2PN3p8uOu492pH5CIJUnEkkTrY1xz2p/pP7wvO++l28FERDpCXeKyUTwa5xeH/pp1K9bTsK6RhnWNxBrj/Oms21nwwaI2l122cDnvvTqLRKzpgCvxxjgP/v7xQpYtIlIWFNiyUe0z75BOp1tMTyaSPHXnC20uu2LJagKhQIvpzsHn85fnrUYRkXKlwJaN6tc1tHgSF0AqmaZuVV2by26z89AWR9cAFUE/X9l/l7zVKCJSrhTYstFX9t+VZKLlld3hbmG+fvQebS5b1b2SE35xDOGq0MZp/gofkW4Rjr3gqLzXKiJSbhTYslG/oX04YVLT0A1XhfjyniMZ883Rm13+pEuO5ad3nMnI3bej37A+jDtlLDfP+B19B/cuZNkiImXBnGvZBdpZ1NTUuNra2lKXUXbeeel9nrz9eRrroow9fm/2O24v/BX+UpclItLlmdl051xNrnm6rUtaGDV2F0aN1XlnEZHORF3iIiIiHqDAFhER8QAFtoiIiAcosEVERDxAgS0iIuIBCmwREREPUGCLiIh4gAJbRETEAxTYIiIiHqDAFhER8QAFtoiIiAcosEVERDwgL4FtZnea2TIze6+V+WZmk81sjpnNNLPNP6tRRERENsrXEfbdwKFtzD8MGJn9mQjclKftdkqNjTGuO/0mfnbgr3jh/qmlLkdERLqAvDxe0zn3ipmNaKPJ0cC9LvPw7TfMrKeZDXLOfZaP7XcmL9w/ld+eNHnj+7dffI/JZ93OwyvuxO/XM6VFRGTrFOsc9hBg0SbvF2endTm/HT+5xbT6tQ38/KDLS1CNiIh0FcUKbMsxzeVsaDbRzGrNrHb58uUFLiu/Xrh/ait7Be9OnVXcYkREpEspVmAvBoZt8n4osCRXQ+fcrc65GudcTb9+/YpSXL6sWrqm1XmZswEiIiJbp1iBPQU4OXu1+J7A2q54/vqIMw5udd7Abbz1x4eIiHQu+bqt62/A68COZrbYzE4zszPM7IxskyeBecAc4DbgzHxst7OJREKMO2W/nPN+8/QlRa5GRES6EuvMXbU1NTWutra21GVssRfun8otP7uP+rX1jNz9S/zy7/9L74G9Sl2WiIh0cmY23TlXk3OeAltERKRzaCuwNTSpiIiIByiwRUREPECBLSIi4gEKbBEREQ9QYIuIiHiAAltERMQDFNgiIiIeoMAWERHxAAW2iIiIByiwRUREPECBLSIi4gEKbBEREQ9QYIuIiHiAAltERMQDFNgiIiIeoMAWERHxAAW2iIiIByiwRUREPECBLSIi4gEKbBEREQ9QYIuIiHiAAltERMQDFNgiIiIeUFHqAqQ4Pnl3Afdc9iAf1c5l0JcGMP7S4/jK/ruWuiwREWknHWGXgY9nzOPcvS7mtcensXzxSma+/AGXHPlbXvnH66UuTURE2kmBXQZuu/A+og0xnHMbp8Ua4vz5vLuaTBMRkc5LXeJdxNx35vPMXS/SWNfIPt/Zi5pDRuHzZf4emz1tbs5l1q5YT92aeqp7dStmqSIishUU2F3AY396ktsn/ZVELEE67Xjp76+x+8GjuPShC/D5fPQe2JOGdY0tlvNX+Ih0C5egYhER2VLqEve4NcvXcuvP/0KsMU46nenejtbHmP7sO0x76i0AvnfRdwhXhposF6oMcsTpB1ER0N9sIiJeoMD2uBnPv0tF0N9ierQ+xtSH32B27VzmzpzP9qO3JRgJEu4WJhgOMO7ksZz+u/ElqFhERLaGDq88LhQJYliL6T6f8cl7C7lg7KXEowlwEIwEGHv81/nxdadSWR0pQbUiIrK1dITtIXPe+oQbf3In15z2Z97813TS6TQ1h4wi13Xe/mAFn7y7iFhDHJd2OOeINcR58f5X+fyTZUWvXUREOkaB7RGP/PEJzvvGJUy58WmeuetFrjzxei4/9loCoQBXTLmQSHWEyuoIkWyX9+4H7Zbzlq1EPMnr/6wtwR6IiEhHqEvcA1YvW8sdv7g/07WdFa2LMv35mUx76i3GHLE7D352G9OefptofZSag0fx8oOvM+Pf77ZYl8/vIxgKFLN8ERHJAx1he8Bbz8/EX5HjwrK6KFMfeROAcGWIfb49hnHj96PXgJ7sc+ye5Oor9/mMfY/bq9Ali4hInimwPSAYCWKW48Iyv49wVSjHEtBnUC8uuOMMguHMleEbrg4/58YfMmCbfoUuWURE8kxd4h7wtUO/Qo4LwQmEKjjk1P1bXe6AE/eh5uCv8MYT03HOMeaI0fTs16OAlYqISKEosD0gFAlxxZRJ/PKoq3AOcI5UMsWEK09k5Ojt2ly2e59qDj5lbFHqFBGRwlFge8Ru++688cKyWEOc0eN2o1d/HS2LiJQLBbaHhCIhvvGtMaUuQ0RESkAXnYmIiHiAAltERMQDFNgiIiIekJfANrNDzWy2mc0xs0k55o81s7Vm9nb259J8bFdERKRcdPiiMzPzAzcC44DFwDQzm+Kc+6BZ06nOuW92dHsiIiLlKB9H2HsAc5xz85xzceAB4Og8rFdERESy8hHYQ4BFm7xfnJ3W3F5m9o6ZPWVmu+Rhu13W/PcXccmRv+WY3qdw6g7n8ORtz+d88paIiJSPfNyHnWPQzBaPnZgBbOOcqzOzw4HHgJE5V2Y2EZgIMHz48DyU5y2LP/6Mc/e6iGh9FOegfk0DN51/N0sXLmfCFSe2uWwinuC1x6Yxu3YuQ7YfyNgT9qaqe2WRKhcRkUKyjh65mdlewP855w7Jvv8FgHPut20sMx+occ6taGvdNTU1rra2vJ7d/PsJN/L8X14hnUo3mR4MB3lo6e1UVkdyLrd+dR3n7nURK5esprEuSrgqRCAU4PpXr2T4Trk6PEREpLMxs+nOuZpc8/LRJT4NGGlm25pZEDgBmNKsgIGWfdyUme2R3e7KPGy7y5n1xkctwhqgIuhnydzPW13u7l8+wOfzl9NYFwUgWh+jbnU9v59wY8FqFRGR4ulwYDvnksDZwDPALOBB59z7ZnaGmZ2RbXYs8J6ZvQNMBk5wOimb09AdBpPjSZokYkn6Dund6nIvP/Q6yXiyyTTnHHNmzKN+XUO+yxQRkSLLy1jizrkngSebTbt5k9c3ADfkY1td3Qm/+BYz/j2TWEN847RgJMDex4xp89GYPn/rf3v5fLkuMxARES/RSGedzM577sAlD5xPv2F9qAhWEAwHGHfyfvz0jh+3udy48fsRDAeaTPP5fez6jZ2IdMt93ltERLxDT+vqhPb85u6MOWI061fVEe4WJhgKbHaZky49lpmvfMD89xeRjCcJhCqo6lHFz+46qwgVi4hIoSmwS8A5x0fT57Hy01VsP3pb+g/r26KNmdG9T3W71xmpCjP5tV8z8+UPmPPWJwzctj9jjhhNRUC/YhGRrkD/Ny+y1cvWcuG4y/ls3lJ8fh+JWJKDT92Pc288HZ+vY2cozIxRY3dh1FiNSyMi0tXoHHaR/ebE61k461Oi9TEa1jWSiCV4/r6pPH3ni6UuTUREOjEFdhGtWb6W91+bTSqZajI91hDj0clPtrKUiIiIAruo5rw9n1QqlXNeg+6VFhGRNugcdgG8/OBr3HHR/SxdsJz+w/ow4coTScSTTD7rdtLJHKOYBfx8/eivlaBSERHxCgV2nr380Ov8/gc3bhz45PP5y7n29JtIJ9MkEy2PrisCfnr078H3Lv5OsUsVEREPUZd4nt1x0V+bjFIGEG9MtDhvvcHQHYdwx3vX0at/66OYiYiI6Ag7z5bOX55zeq6R083gS18ZQVWPqlaWccyeNodP3l3I0B0Gs+s3dsJyDTQuIiJdngI7z/oP78vnnyxrMd3MaP68k2AkxGGnHZBzPdGGGL847ErmzPhk4/IDt+vPb5+6mMdveJqnbv838WiCPQ7/KhN/fzL9hvbJ/86IiEinoS7xPJtw5YmEKkNNpoUqQ5ww6RiqelRSWR0hXBUmEA5w3E+PYtR+uQc5uevivzH7v3OJ1seI1sdorIuyaNannDH65zz8hydYs3wdDesbeeUfb3DW1y6kfm19MXZPRERKREfYeXbAid/AOcedv7ifZYtW0HdIb079vzGM++4yvve/+zLtpeE01FUw+qDd2jwqfvbel0jEEk2mJRMp1ixd22RaOpWmYX2UZ+5+iW//5IiC7JOIiJSeArsADvzePhz4vX1Ip5Ow9lyIT4b6BoKE2HsfH9bzBizUdhd282dbtyXWEOPDNz/uaNkiItKJqUu8AJxzOJfGYk9D/D/gNgyKEgPXiFtzHs7F21zHHoePzvmMa8vxbOtgOMCIXYflo3QREemkFNh55NJrSa+5ALd0V9zSnXHrfgWuMVdLiL/V5rrOuPYUevTrTrgqcz48FAlS1bOSoTsMpiLgb9K2IljBYacdmK/dEBGRTkhd4nninMOtGg/JuUD23LNb2/oC9sXfSi72Kq7+Dkgtg9C+WNVp9Bval7tnT+b5+17hw2kfs+2uwzlkwv4AXP+jW3h9Si3ptGP7r27L+bedQa8BPQu4dyIiUmrW/FajzqSmpsbV1taWuox2cfFpuNWnb9L93QbrgfV/HbMK0vX3Qd01mxyJB8DXHevzBOZv/Tx3Ip4glUwTbnZFuoiIeJeZTXfO1eSapy7xfEnOA9dynPAMHxAAKsEqsV5/zoR1eh2s/12zbvMEpNfhGu5oc3OBYEBhLSJSRtQlni8VIzPd3C06LCJQeTLm7wnWA8KHYL5qXPR5WPNTIJZjZQmITYXqnxe+bhER8QQFdr4EvpoJ7cQsYMMV4D7wVWLdfoT5um1s6pLzcWvOB6Ktr8/Xv5DVioiIx5R1l/jqZWv55N0FxBpzHeVuGTPDet0Nke+AVQEhCB2I9Xm4SVgDuIYHgLbus45gVad1uCYREek6yvIIu7GukavG/4lpT79NIFhB2jkmXHE83/7JNzu0XvNVYT1+BT1+1XbD9FJaD2wfdDsTC329Q7WIiEjXUpZH2L//wZ+Z9vTbJGIJGtY3Eq2LcufFD/Da49OKU0Cb3d0OGu4lnVxQnFpERMQTyi6w161azxv/nN5inO5YQ4wHrn604Nt36VXQcH9bLSC9HFYcRnr5objEBwWvSUREOr/yC+yVdfibjRS2wcolqwtfQPRZ2vexJyE1D7fqJFy6jQFYRESkLJRdYA8c0Q9/Rcvd9vl9jBqb+1GXeeUaaPuCs+btUxB9omDliIiIN5RdYFcEKjjj2lOaPLPaX+EjUh1m/GXHFb6A0L5s2cfeiEsuASCVTPHyQ69z9cl/4qbz72bBB4sKUqKIiHQ+ZXmV+KETDqD/8H48cNWjLFu4nN3224XvX/wdBmzTr+DbtortcZUnQsPfydyH7cgEuCPHqCuZkdGCXyURT3DhuCv4eMY8ovUxfH4f/7rlOc679Ucc9P19C163iIiUlsYSLwHnHMT/i4s+lhnONHwIJJdC/fXg1gGpbMsQVGyH9XmYZ++Zyg3n3EG0vuk946HKEA8tvZ1IVbjo+yEiIvnV1ljiZXmEXWpmBqExWGgMLt2AW3UcJBcDm44p3g0qx2PdJmJWwYsP/KdFWANUVPh5/z+zqTl4VNHqFxGR4iu7c9idjWu4H5ILaRrWAHUQfxUsc0V7pFvuI+i0S298ZraIiHRdCuxSiz5F7geAAMnZuPp7APjmj8blDOZItzBf3nNkAQsUEZHOQIFdar7qNmbGofFxAHYfN4pvn3cEwXCASLcwld0jdO9Tza//dRF+f+77ykVEpOvQOewSs8qTcPEZtPrkLvsijCdccSJHnnEwb7/4PtW9qhg9bjcCwUBxChURkZJSYJda6ECoPAka7qDlbV1hiDS9N7zvkD4cdJJu4xIRKTcK7AJzqeUQnwqEIbQf5qtqMt/MoPpnOP8IWH8lmdBOAQEIfg2rPLH4RYuISKejwC6gdP3dsP4asArAAAc9b8RCe+PSdbh4LcTfhMZHwEUzbawCwt/BIt+CwG6ZQBcRkbKnwN5CzqUh/h9c7DXw9cYiR2P+lo/LdIlZsP46IA4u/sX01WfiIidA4718MUDKpgsC0cdwVWdhiZk4YhAYhZlu3RIRKWcK7C3gXAK3+nRIvJ19iEcQV3cD9PozBL8GyXng64n5B+IaHwPiOdaSbD2sN24oASsPxZFiw7ClrvtV+CKHFGK3RETEAxTYW6LxMYi/xReDnGQC2a0+E/Ble70TuOBo8A0C0jlWkiTnmOFNJDKhvam1P8MFdsIqtunADoiIiFfpPuwtkDlqbj4iGdlp9eDqgTjEayHxHhDJtZat3HoS1/jQVi4rIiJep8DeEtbeDokkpBZC4Gt8EdrtvXisgty/liSkV7ZzHSIi0tXkJbDN7FAzm21mc8xsUo75ZmaTs/NnmtnofGy3UJxzuOQnuMTszEVmWRY5jtxHzbnEIDEVSIJvOO0L7EFQdRY5z1RYJRYa285ti4hIV9Phc9hm5gduBMYBi4FpZjbFOffBJs0OA0Zmf8YAN2X/7XRccl7mnHRqCZgPLAI9rsNCe0H4cIhNzY7/nQYLZK8AT7S2tsy89MJ2bdsGPItZiLRbBw0PsLH73SJQ8eXMICsiIlKW8nHR2R7AHOfcPAAzewA4Gtg0sI8G7nWZh2+/YWY9zWyQc+6zPGw/b5xL4FadlO16dpm8dQ24NWdA32cw/0Cs59W4xA8y90/7euGsCtackZ/tNzyIVY3HqifhAntAw12Z+7Mjx2KV38ba3SUvIiJdTT66xIcAizZ5vzg7bUvblF5sKrhGWlwY5lK4hn9sfGuBHbGqkyEwCtZfm7/tr78aF58GydlQ91tIvAvJj6H+JkjMyt92RETEc/JxyJbr5GzzS6Hb0ybT0GwiMBFg+PDhHatsS6WXg8t1K1YcUvObTHHpetzKo7NXhudLHFd3FySmgVu7SV2NuNWnQr8XMV/PPG5PRES8Ih9H2IuBYZu8Hwos2Yo2ADjnbnXO1Tjnavr165eH8rZAcHdave0q+iTpVRNx6fUAuOiULQxrH1ivzTdLzSXnOXGXgsYntmB7IiLSleQjsKcBI81sWzMLAicAU5q1mQKcnL1afE9gbWc7fw1gFdtD+BByXwmehPhruDXn4ZILsheFbdHawa3bfDP/iCZDmX4hiksv28JtiohIV9HhwHbOJYGzgWeAWcCDzrn3zewMM9twNdaTwDxgDnAbcGZHt1so1uNqrPtlYN1yzI1nQnvFEZnzzO3mz/60MRwpAFVQdQqQ4+/sgMAAABBTSURBVBnXVokFa7ZgmyIi0pVY5sLtzqmmpsbV1tYWfbvOJXFL9wDqirth6wuBXSG9FJKfANHsjDAEdsF6/xUzjXUjItJVmdl051zOozPdJ9SMSy7GrTqFooc1gFsB8ZeAMPgGgy8EpCH8LazqJIW1iEgZK5vATsQTLP7oM7r3qabPoNwXf7l0HW7lseBWFbm65qKQXgBVl+OrOq7EtYiISGdQFoH9zN0v8ufz7sI5RzKeYrd9v8zFD/wv1b2anaeOPpG9D7szSMH6y3DB3bDAjqUuRkRESqzL97HOfOUD/nT2HTSsa6RxfZRELME7L3/Ar75zTYu2LvERuZ/GVSpJ3Ppfl7oIERHpBLp8YD90zRRiDbEm05LxJLPe+IilC5Y3mW6BLwOVRayuHeIzSl2BiIh0Al0+sJctWpFzekWwglWfr2k6MXwE+KroVB9L9vayFZ+uZOYrH7B66ZrNLCAiIl1Rlz+Hvfu43Vg461OS8WST6alEim12aCS9/obMQzXCh2IVI6DPw7i1V0D8xQ0taXX0syJIhb7Hld+9ljefmE4gFCAeTXDQSfvwk5sn4vf7S1aXiIgUVyc6lCyMY88/kqruEfyBL8ItXBli/EWDCMdOgvobcXWTcSuOJF3/F/D1ALeSzOAlpQ1rKnbi1ssivPmv6cSjCerXNpCIJXjhb6/y96sfK11dIiJSdGUxcMqKJat44KpHqX3mHXoN7MFx541mz70u44uBSTYIQeQYaHyE1p9xXWhBwEFoX1z36zm6+w+INbYcqrRn/x489PntxS9PREQKpuwHTuk7uDdnTz5t4/v0+j9Afa7xug0ap1C6sI5A1SlY5QmYfzCJeIJ4LHctDesailybiIiUUlkEdkt+cj/xs/kRd5GZYd3OwiwEQCAYYMQuw/jk3YUtmu78dd2bLSJSCuN8XwxoNfqQXbn6qcuKst0ufw47FwsfRs4HbJRaxW4bw3qDc/98OqHKED5/5lflr/ATqQ5z5h9OLUGBIiLlbdOwBpjxzHstphVKeQZ2YCR0O5vMkXYn4m/5/O9d996JG6ddxbjx+7LD7ttx+A8P5Ja3r2Hb/9mmBAWKiJSvtoL5L1f+veDbL4uLznJxLopbWgPkOpddIuHjsR6XZW4zExGRTmVzR9LPpR/q8DbauuisLI+wAUgt33ybYos+jlv781JXISIinVD5Bra/H5AudRXNRCH6HC7Z8iIzERHpvAZu37fg2yjbwDYLQ2hsqctoyQKQ/LDUVYiISDOnXH5sq/Pu++imgm+/bAMbgOorSl1BSy4J/qGlrkJERJo56ZLjOefWH7SYno9z1+1R3lc3NRb+qr4t5h+GBXYudRUiIpLDUT88jKN+eFhJtl3eR9iN/2hjZq6BVTqiksywo5vR80953q6IiHQF5X2EnW7rSvE83u7m2x7Si8g8TKQtFdgmo6011jXy/F+m8tH0uWy763DGnbwf1b265a8uERHxjPIObOsOrpC3d1WA9YT0YiDWnoLAlxk8ZcWnKzlrj1/QsK6BaH2MUCTIfZc/xOTXfs2wHYcUsGYREemMyrtLvOr7tKubequlwK2ifWOUByC0D5Yd7eyWn97LmmVridZngj7WGKd+TQN/mHhL4coVEZFOq6wD26pOh9B+QIjMOeYI+AZn/s0Lx+bv9TYyYT0W63HNxqlvPDGddKrpss453n9tNol4qZ4mJiIipVLWXeJmAazXjbjkXEh8CP6huIpdYd3PIPpvMo/Z9NO+7uwtFYCeN0F6DUQfh9RCXN0foOp0zD+QimAF1Lfcrvls44NARESkfOj//IBVfAmLHIEFR+Hz+fH1vA7r8xeInEDmCDjf3eZ+qL4I3DpYdwnEp0JyNjT8DbfiSFzqU8aN349AqOkTxSoCfr5+9Nfw+zvZQ0tERKTgFNitqdgJok+QOf/c7AEhFXuT6UbfWg7Ch8G6y2l6fjsJrg5XdwMTfn0iI3ffjnBViFBliEi3MEN2GMx5N03swHZFRMSryrpLvE3x/5LpEs8h+Z82FvSR6UZ3QLKVNmmI/Qdcrq72FMReI9IjzPVTr2DWmx/zycwFDN1hMLvttzNm+b4/XEREvECB3RrXyBYPnhI8GgIjsMiRkFqMWz2BVu/n9vWj1fuyfb0BMDN23nMHdt5zhy2rQ0REuhwFdmuCe4DbwquxI4fgixyUeV0xHOffBlLzczT0YaHRuNC+EHuFpl3ukczV6yIiIpvQOexWmK87dL+ELfubpultWFb9M1pesOaDyHjMgliP30FwDBAC6waEoduPsMjhHapdRES6Hh1ht8FXeTzp+AyIPtqe1hA6qMkUC4/Ddb8M1l8Drg7wQ+V4rPr8zHxfN6z3HbjU55BaChVfwnwaelRERFpSYG+GVR6Liz2dPafdhu5X4vO17LDwVR6Hi3wH3Bqwbpi1vEXM/APBPzBfJYuISBekwN6cQA2EDoDYi+AashMNfMPAPwD8g6Db+fgqBre6CjMfWO/i1CsiIl2SAnszzAx6XAuxl3DRKUAAi3wbC+1V6tJERKSMKLCbmTdzAR/+dw79h/flqwfuit/vzxwhhw/AwgeUujwRESlTCuysZCLJ5cddy4znZwKGz290713Nda9cTv9hfUtdnoiIlDkFdtYjf3ySGc/PJNbwxT3RsYY4v/ne9Vw/9coW7T95byGP/OEJPp3zObuN3YVjzj6Unv16kEqlmPnyByz8YDFzZy5gwfuLGL7TEI694Ei22XlYMXdJRES6EHOulZG4OoGamhpXW1tblG2duuM5fPrx5y2mB0IV/G3RLfTo233jtGlPv8Wvjr2GRCxJOpUmEApQWR3h/x79Gb/9/h9Zt3L9xudYQ+YJW6FwkCv+OYmv7L9rUfZHRES8x8ymO+dqcs3TwClZ8WjuUc3MjETsi3nOOa47/WZiDfGNz6tOxBLUranjl0dexfLFK5uENYBLO6INMa7/0S105j+QRESk81JgZ+177J4Egi3PEPQb2oc+g7+4JWvFp6tYt6quRbtUMk3dmnpcuvVAXrpgOfVrG1qdLyIi0hoFdtb3LzmWfsP7Eq4KAxAMB4h0C3Phfec2eUJWZXV445F1c+Zr+2Eh5vMRjOT72doiIlIOdNFZVnWvbtw281pe+vtrzHzlA4ZsP5BDJuxP74G9mrSr6lFFzcGjqH32HZLxLx6fGYwE8fmsRXf4BoFQgP2+uxfBUKCg+yEiIl2TLjrbCutX1/HLo65mzox5VAQriEcTHDHxIHYfN4orT7iOZCJFKpF5dKb5jIpgBaP225lLH7qASLdIiasXEZHOqq2LzjoU2GbWG/g7MAKYD3zXObc6R7v5wHoyD4BOtlZMc501sDdYMGsxyxet5EujtqHXgJ4AfDrnM/5163MsX7SSncaMZNhOQxi2w2AGbTegxNWKiEhnV8jA/h2wyjl3lZlNAno55y7M0W4+UOOcW7El6+/sgS0iIpJPhbyt62jgnuzre4BjOrg+ERERyaGjgT3AOfcZQPbf/q20c8CzZjbdzCZ2cJsiIiJlZ7NXiZvZ80CuhzVfvAXb2ds5t8TM+gPPmdmHzrlXWtneRGAiwPDhw7dgEyIiIl3XZgPbOXdQa/PMbKmZDXLOfWZmg4BlraxjSfbfZWb2KLAHkDOwnXO3ArdC5hz25ndBRESk6+tol/gU4JTs61OAx5s3MLMqM6ve8Bo4GHivg9v1tFQqxZv/ms5frvgHz//lFWKNue/dFhER2aCjA6dcBTxoZqcBC4HjAMxsMHC7c+5wYADwaHa0sArgfufc0x3crmc1rG/kf/f9JZ/NXUq0Pka4KsQtP72H61+9kiHbDyp1eSIi0kl1KLCdcyuBA3NMXwIcnn09DxjVke10Jff+6kEWfbhk4wNFGuuiRBtiXH3yDUx+7dclrk5ERDorjSVeZC/c/2qTp39B5mleH0+fS/3a+hJVJSIinZ0Cu8jafjyIiIhIbgrsIjvg+/sQCDU9E2E+Y4eaL1HVo6pEVYmISGenwC6y8Zcex/AvDyXSLYyZEekWpkffan5+z9mlLk1ERDoxPV6zyCqrI9w47Spqn3mHOW99wsAR/fnGt/cgFAmVujQREenEFNgl4Pf7GXP4aMYcPrrUpYiIiEeoS1xERMQDFNgiIiIeoMAWERHxAAW2iIiIByiwRUREPECBLSIi4gEKbBEREQ9QYIuIiHiAAltERMQDFNgiIiIeoMAWERHxAHPOlbqGVpnZcmBBqetoh77AilIXUULlvv+gz6Dc9x/0GZT7/kN+PoNtnHP9cs3o1IHtFWZW65yrKXUdpVLu+w/6DMp9/0GfQbnvPxT+M1CXuIiIiAcosEVERDxAgZ0ft5a6gBIr9/0HfQblvv+gz6Dc9x8K/BnoHLaIiIgH6AhbRETEAxTYW8HMepvZc2b2cfbfXq20m29m75rZ22ZWW+w6883MDjWz2WY2x8wm5ZhvZjY5O3+mmY0uRZ2F0o79H2tma7O/77fN7NJS1FkoZnanmS0zs/damd+lf//Qrs+gq38HhpnZi2Y2y8zeN7Of5GjTZb8H7dz/wn0HnHP62cIf4HfApOzrScDVrbSbD/Qtdb152mc/MBfYDggC7wA7N2tzOPAUYMCewJulrrvI+z8WeKLUtRbwM9gXGA2818r8Lvv734LPoKt/BwYBo7Ovq4GPyuz/A+3Z/4J9B3SEvXWOBu7Jvr4HOKaEtRTLHsAc59w851wceIDM57Cpo4F7XcYbQE8zG1TsQgukPfvfpTnnXgFWtdGkK//+gXZ9Bl2ac+4z59yM7Ov1wCxgSLNmXfZ70M79LxgF9tYZ4Jz7DDK/QKB/K+0c8KyZTTeziUWrrjCGAIs2eb+Yll/U9rTxqvbu215m9o6ZPWVmuxSntE6jK//+t0RZfAfMbATwVeDNZrPK4nvQxv5Dgb4DFflaUVdjZs8DA3PMungLVrO3c26JmfUHnjOzD7N/oXuR5ZjW/BaD9rTxqvbs2wwywwrWmdnhwGPAyIJX1nl05d9/e5XFd8DMugEPA+c559Y1n51jkS71PdjM/hfsO6Aj7FY45w5yzu2a4+dxYOmGLp7sv8taWceS7L/LgEfJdKt61WJg2CbvhwJLtqKNV21235xz65xzddnXTwIBM+tbvBJLriv//tulHL4DZhYgE1Z/dc49kqNJl/4ebG7/C/kdUGBvnSnAKdnXpwCPN29gZlVmVr3hNXAwkPPKUo+YBow0s23NLAicQOZz2NQU4OTsVaJ7Ams3nDroAja7/2Y20Mws+3oPMv99rSx6paXTlX//7dLVvwPZfbsDmOWcu66VZl32e9Ce/S/kd0Bd4lvnKuBBMzsNWAgcB2Bmg4HbnXOHAwOAR7O/twrgfufc0yWqt8Occ0kzOxt4hswV03c65943szOy828GniRzhegcoAGYUKp6862d+38s8GMzSwKNwAkue9loV2BmfyNzBWxfM1sMXAYEoOv//jdox2fQpb8DwN7AeOBdM3s7O+0iYDiUxfegPftfsO+ARjoTERHxAHWJi4iIeIACW0RExAMU2CIiIh6gwBYREfEABbaIiIgHKLBFREQ8QIEtIiLiAQpsERERD/h/XN4oC0ve8uQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "### COMPUTE ISOLATION FOREST FOR ANOMALY SENTENCES DETECTION ###\n",
    "\n",
    "IF = IsolationForest(contamination=0.3)\n",
    "IF.fit(X)\n",
    "\n",
    "plt.figure(figsize=(8,6))\n",
    "plt.scatter(X.T[0], X.T[1], c=IF.predict(X), cmap='viridis')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Shepard Convolutional Neural Networks',\n",
       " 'Abstract\\nDeep learning has recently been introduced to the field of low-level computer\\nvision and image processing',\n",
       " 'Promising results have been obtained in a number of tasks including super-resolution, inpainting, deconvolution, filtering, etc',\n",
       " 'However, previously adopted neural network approaches such as convolutional\\nneural networks and sparse auto-encoders are inherently with translation invariant\\noperators',\n",
       " 'We found this property prevents the deep learning approaches from\\noutperforming the state-of-the-art if the task itself requires translation variant interpolation (TVI)',\n",
       " 'In this paper, we draw on Shepard interpolation and design\\nShepard Convolutional Neural Networks (ShCNN) which efficiently realizes endto-end trainable TVI operators in the network',\n",
       " 'We show that by adding only a few\\nfeature maps in the new Shepard layers, the network is able to achieve stronger\\nresults than a much deeper architecture',\n",
       " 'Superior performance on both image inpainting and super-resolution is obtained where our system outperforms previous\\nones while keeping the running time competitive',\n",
       " '\\n1 Introduction\\nIn the past a few years, deep learning has been very successful in addressing many aspects of visual\\nperception problems such as image classification, object detection, face recognition [1, 2, 3], to name\\na few',\n",
       " 'Inspired by the breakthrough in high-level computer vision, several attempts have been made\\nvery recently to apply deep learning methods in low-level vision as well as image processing tasks',\n",
       " 'Encouraging results has been obtained in a number of tasks including image super-resolution [4],\\ninpainting [5], denosing [6], image deconvolution [7], dirt removal [8], edge-aware filtering [9] etc',\n",
       " 'Powerful models with multiple layers of nonlinearity such as convolutional neural networks (CNN),\\nsparse auto-encoders, etc',\n",
       " 'were used in the previous studies',\n",
       " 'Notwithstanding the rapid progress and\\npromising performance, we notice that the building blocks of these models are inherently translation\\ninvariant when applying to images',\n",
       " 'The property makes the network architecture less efficient in\\nhandling translation variant operators, exemplified by the image interpolation operation',\n",
       " 'Figure 1 illustrates the problem of image inpainting, a typical translation variant interpolation (TVI)\\ntask',\n",
       " 'The black region in figure 1(a) indicates the missing region where the four selected patches\\nwith missing parts are visualized in figure 1(b)',\n",
       " 'The interpolation process for the central pixel in\\neach patch is done by four different weighting functions shown in the bottom of figure 1(b)',\n",
       " 'This\\nprocess cannot be simply modeled by a single kernel due to the inherent spatially varying property',\n",
       " 'In fact, the TVI operations are common in many vision applications',\n",
       " 'Image super-resolution, which\\naims to interpolate a high resolution image with a low resolution observation also suffers from the\\n∗',\n",
       " '(b)\\nFigure 1: Illustration of translation variant interpolation',\n",
       " '(a) The application of inpainting',\n",
       " 'The black regions\\nindicate the missing part',\n",
       " '(b) Four selected patches',\n",
       " 'The bottom row shows the kernels for interpolating the\\ncentral pixel of each patch',\n",
       " '\\nsame problem: different local patches have different pattern of anchor points',\n",
       " 'We will show that it\\nis thus less optimal to use the traditional convolutional neural network to do the translation variant\\noperations for super-resolution task',\n",
       " 'In this paper, we draw on Shepard method [10] and devise a novel CNN architecture named Shepard Convolutional Neural Networks (ShCNN) which efficiently equips conventional CNN with the\\nability to learn translation variant operations for irregularly spaced data',\n",
       " 'By adding only a few\\nfeature maps in the new Shepard layer and optimizing a more powerful TVI procedure in the endto-end fashion, the network is able to achieve stronger results than a much deeper architecture',\n",
       " 'We\\ndemonstrate that the resulting system is general enough to benefit a number of applications with TVI\\noperations',\n",
       " '\\n2 Related Work\\nDeep learning methods have recently been introduced to the area of low-level computer vision and\\nimage processing',\n",
       " '[6] used a simple multi-layer neural network to directly learn a\\nmapping between noisy and clear image patches',\n",
       " '[5] adopted a sparse auto-encoder and\\ndemonstrated its ability to do blind image inpainting',\n",
       " 'A three-layer CNN was used in [8] to tackle\\nof problem of rain drop and dirt',\n",
       " 'It demonstrated the ability of CNN to blindly handle translation\\nvariant problem in real world challenges',\n",
       " '[7] advocated the use of generative approaches to guide the design of the CNN for deconvolution tasks',\n",
       " 'In [9], edge-aware filters can be well approximated using CNN',\n",
       " 'While it is feasible\\nto use the translation invariant operators, such as convolution, to obtain the translation variant results\\nin a deep neural network architecture, it is less effective in achieving high quality results for interpolation operations',\n",
       " 'The first attempt using CNN to perform image super-resolution [4] connected\\nthe CNN approach to the sparse coding ones',\n",
       " 'But it failed to beat the state-of-the-art super resolution system [11]',\n",
       " 'In this paper, we focus on the design of deep neural network layer that better fits\\nthe translation variant interpolation tasks',\n",
       " 'We note that TVI is the essential step for a wide range of\\n2',\n",
       " '\\x0clow-level vision applications including inpainting, dirt removal, noise suppression, super-resolution,\\nto name a few',\n",
       " '\\n3 Analysis\\nDeep learning approaches without explicit TVI mechanism generated reasonable results in a few\\ntasks requiring translation variant property',\n",
       " 'To some extent, deep architecture with multiple layers of\\nnonlinearity is expressive to approximate certain TVI operations given sufficient amount of training\\ndata',\n",
       " 'It is, however, non-trivial to beat non-CNN based approaches while ensuring the high efficiency\\nand simplicity',\n",
       " 'To see this, we experimented with the CNN architecture in [4] and [8] and trained a CNN with three\\nconvolutional layers by using 1 million synthetic corrupted/clear image pairs',\n",
       " 'Network and training\\ndetails as well as the concrete statistics of the data will be covered in the experiment section',\n",
       " 'Typical\\ntest images are shown in the left column of figure 2 whereas the results of this model are displayed\\nin the mid-left column of the same figure',\n",
       " 'We found that visually very similar results as in [5] are\\nobtained, namely obvious residues of the text are still left in the images',\n",
       " 'We also experimented with\\na much deeper network by adding more convolutional layers, virtually replicating the network in\\n[8] by 2,3, and 4 times',\n",
       " 'Although slight visual differences are found in the results, no fundamental\\nimprovement in the missing regions is observed, namely residue still remains',\n",
       " 'A sensible next step is to explicitly inform the network about where the missing pixels are so that\\nthe network has the opportunity to figure out more plausible solutions for TVI operations',\n",
       " 'For many\\napplications, the underlying mask indicating the processed regions can be detected or be known\\nin advance',\n",
       " 'Sample applications include image completion/inpainting, image matting, dirt/impulse\\nnoise removal, etc',\n",
       " 'Other applications such as sparse point propagation and super resolution by\\nnature have the masks for unknown regions',\n",
       " 'One way to incorporate the mask into the network is to treat it as an additional channel of the input',\n",
       " 'We tested this idea with the same set of network and experimental settings as the previous trial',\n",
       " 'The results showed that such additional piece of information did bring about improvement but still\\nconsiderably far from satisfactory in removing the residues',\n",
       " 'Results are visualized in the mid-right\\ncolumn of figure 2',\n",
       " 'To learn a tractable TVI model, we devise in the next session a novel architecture\\nwith an effective mechanism to exploit the information contained in the mask',\n",
       " '\\n4 Shepard Convolutional Neural Networks\\nWe initiate the attempt to leverage the traditional interpolation framework to guide the design of\\nneural network architecture for TVI',\n",
       " 'We turn to the Shepard framework [10] which weighs known\\npixels differently according to their spatial distances to the processed pixel',\n",
       " 'Specifically, Shepard\\nmethod can be re-written in a convolution form\\nJp =',\n",
       " 'Mp = 0 indicates the pixel values are unknown',\n",
       " '∗ is the convolution operation',\n",
       " 'K is the kernel function with its weights inversely proportional to the distance between a pixel\\nwith Mp = 1 and the pixel to process',\n",
       " 'The element-wise division between the convolved image\\nand the convolved mask naturally controls the way how pixel information is propagated across the\\nregions',\n",
       " 'It thus enables the capability to handle interpolation for irregularly-spaced data and make\\nit possible translation variant',\n",
       " 'The key element in Shepard method affecting the interpolation result\\nis the definition of the convolution kernel',\n",
       " 'We thus propose a new convolutional layer in the light of\\nShepard method but allow for a more flexible, data-driven kernel design',\n",
       " 'The layer is referred to as\\nthe Shepard interpolation layer',\n",
       " '\\x0cFigure 2: Comparison between ShCNN and CNN in image inpainting',\n",
       " 'Input images (Left)',\n",
       " 'Results from a\\nregular CNN (Mid-left)',\n",
       " 'Results from a regular CNN trained with masks (Mid-right)',\n",
       " 'Our results (Right)',\n",
       " '\\n4.1 The Shepard Interpolation Layer\\nThe feed-forward pass of the trainable interpolation layer can be mathematically described as the\\nfollowing equation,\\nX Knij ∗ Fjn−1\\nn\\nFin (F n−1 , Mn ) = σ(\\nn ∗ Mn + b ),\\nK\\nj\\nij\\nj',\n",
       " 'The subscript i in Fin is the index of feature maps in layer n',\n",
       " 'F n−1 and Mn are the input and the mask of the\\ncurrent layer respectively',\n",
       " 'F n−1 represents all the feature maps in layer n − 1',\n",
       " 'Kij are the trainable\\nkernels which are shared in both numerator and denominator in computing the fraction',\n",
       " 'Concretely,\\nsame Kij is to be convolved with both the activations of the last layer in the numerator and the\\nmask of the current layer Mn in the denominator',\n",
       " 'F n−1 could be the output feature maps of regular\\nlayers in a CNN such as a convolutional layer or a pooling layer',\n",
       " 'It could also be a previous Shepard\\ninterpolation layer which is a function of both F n−2 and Mn−1 ',\n",
       " 'Thus Shepard interpolation layers\\ncan actually be stacked together to form a highly nonlinear interpolation operator',\n",
       " 'b is the bias\\nterm and σ is the nonlinearity imposed to the network',\n",
       " 'F is a smooth and differentiable function,\\ntherefore standard back-propagation can be used to train the parameters',\n",
       " 'Figure 3 illustrates our neural network architecture with Shepard interpolation layers',\n",
       " 'The inputs of\\nthe Shepard interpolation layer are images/feature maps as well as masks indicating where interpolation should occur',\n",
       " 'Note that the interpolation layer can be applied repeatedly to construct more\\ncomplex interpolation functions with multiple layers of nonlinearity',\n",
       " 'The mask is a binary map of\\nvalue one for the known area, zero for the missing area',\n",
       " 'Same kernel is applied to the image and\\nthe mask',\n",
       " 'We note that the mask for layer n + 1 can be automatically generated by the result of\\nprevious convolved mask Kn ∗ Mn , by zeroing out insignificant values and thresholding it',\n",
       " 'It is\\nimportant for tasks with relative large missing areas such as inpainting where sophisticated ways of\\npropagation may be learned from data by multi-stage Shepard interpolation layer with nonlinearity',\n",
       " 'This is also a flexible way to balance the kernel size and the depth of the network',\n",
       " '\\x0cFigure 3: Illustration of ShCNN architecture for multiple layers of interpolation',\n",
       " '\\na convolutional neural network with Shepard interpolation layers as Shepard convolutional neural\\nnetwork (ShCNN)',\n",
       " '4.2 Discussion\\nAlthough standard back-propagation can be used, because F is a function of both Ks in the fraction, matrix form of the quotient rule for derivatives need to be used in deriving the back-propagation\\nequations of the interpolation layer',\n",
       " 'To make the implementation efficient, we unroll the two convolution operations K ∗ F and K ∗ M into two matrix multiplications denoted W · I and W · M\\nwhere I and M are the unrolled versions of F and M',\n",
       " 'W is the rearrangement of the kernels where\\neach kernel is listed in a single row',\n",
       " 'E is the error function to compute the distance between the\\nnetwork output and the ground truth',\n",
       " 'L2 norm is used to compute this distance',\n",
       " 'We also denote\\nn\\nn−1\\n∂E\\nn\\nn\\nZ n = KKn∗F\\n∗Mn ',\n",
       " 'The derivative of the error function E with respect to Z , δ = ∂Z n , can be\\ncomputed the same way as in previous CNN papers [12, 1]',\n",
       " 'Once this value is computed, we show\\nthat the derivative of E with respect to the kernels W connecting j th node in (n − 1)th layer to ith\\nnode in nth layer can be computed by,\\nn\\nn\\nX (Wij\\n· Mjm ) · Ijm − (Wij\\n· Ijm ) · Mjm\\n∂E\\n· δim ,\\n=\\nn\\nn\\n2\\n∂Wij\\n(Wij · Mjm )\\nm',\n",
       " 'where m is the column index in I, M and δ',\n",
       " 'The denominator of each element in the outer summation in Eq',\n",
       " '3 is different',\n",
       " 'Therefore, the\\nnumerator of each summation element has to be computed separately',\n",
       " 'While this operation can still\\nbe efficiently parallelized by vectorization, it requires significantly more memory and computations\\nthan the regular CNNs',\n",
       " 'Though it brings extra workload in training, the new interpolation layer only\\nadds a fraction of more computation during the test time',\n",
       " '2, the only\\nadded operations are the convolution of the mask with the K and the point-wise division',\n",
       " 'Because\\nthe two convolutions shares the same kernel, it can be efficiently implemented by convolving with\\nsamples with the batch size of 2',\n",
       " 'It thus keeps the computation of Shepard interpolation layer\\ncompetitive compare to the traditional convolution layer',\n",
       " 'We note that it is also natural to integrate the interpolation layer to any previous CNN architecture',\n",
       " 'This is because the new layer only adds a mask input to the convolutional layer, keeping all other\\ninterfaces the same',\n",
       " 'This layer can also degenerate to a fully connected layer because the unrolled\\nversion of Eq',\n",
       " '2 merely contains matrix multiplication in the fraction',\n",
       " 'Therefore, as long as the TVI\\noperators are necessary in the task, no matter where it is needed in the architecture and the type of\\nlayer before or after it, the interpolation layer can be seamlessly plugged in',\n",
       " '\\x0cLast but not least, the interpolation kernels in the layer is learned from data rather than hand-crafted,\\ntherefore it is more flexible and could be more powerful than pre-designed kernels',\n",
       " 'On the other\\nhand, it is end-to-end trainable so that the learned interpolation operators are embedded in the overall\\noptimization objective of the model',\n",
       " '\\n5 Experiments\\nWe conducted experiments on two applications involving TVI: the inpainting and the superresolution',\n",
       " 'The training data was generated by randomly sampling 1 million patches from 1000\\nnatural images scraped from Flickr',\n",
       " 'Grayscale patches of size 48x48 were used for both tasks to\\nfacilitate the comparison with previous studies',\n",
       " 'All PSNR comparison in the experiment is based on\\ngrayscale results',\n",
       " 'Our model can be directly extended to process color images',\n",
       " '5.1 Inpainting\\nThe natural images are contaminated by masks containing text of different sizes and fonts as shown\\nin figure 2',\n",
       " 'We assume the binary masks indicating missing regions are known in advance',\n",
       " 'The\\nShCNN for inpainting is consists of five layers, two of which are Shepard interpolation layers',\n",
       " 'We\\nuse ReLU function [1] to impose nonlinearity in all our experiments',\n",
       " '4x4 filters were used in the\\nfirst Shepard layer to generate 8 feature maps, followed by another Shepard interpolation layer with\\n4x4 filters',\n",
       " 'The rest of the ShCNN is conventional CNN architecture',\n",
       " 'The filters for the third layer is\\nwith size 9x9x8, which are use to generate 128 feature maps',\n",
       " '8x8 filters are used to carry out the reconstruction of image details',\n",
       " 'Visual results are shown\\nin the last column in figure 2',\n",
       " 'More examples are provided in the project webpage',\n",
       " '\\n(a) Ground Truth / PSNR',\n",
       " 'Figure 4: Visual comparison',\n",
       " 'Factor 4 upscaling of the butterfly image in Set5 [14]',\n",
       " '\\n5.2 Super Resolution\\nThe quantitative evaluation of super resolution is conducted using synthetic data where the high\\nresolution images are first downscaled by a factor to generate low resolution patches',\n",
       " 'To perform\\nsuper resolution, we upscale the low resolution patches and zero out the pixels in the upscaled\\nimages, leaving one copy of pixels from low resolution images',\n",
       " 'In this regard, super resolution can\\nbe seemed as a special form of inpainting with repeated patterns of missing area',\n",
       " '\\x0cSet14 (x2)\\nbaboon\\nbarbara\\nbridge\\ncoastguard\\ncomic\\nface\\nflowers\\nforeman\\nlenna\\nman\\nmonarch\\npepper\\nppt3\\nzebra\\nAvg PSNR\\nSet14 (x3)\\nbaboon\\nbarbara\\nbridge\\ncoastguard\\ncomic\\nface\\nflowers\\nforeman\\nlenna\\nman\\nmonarch\\npepper\\nppt3\\nzebra\\nAvg PSNR\\nSet14 (x4)\\nbaboon\\nbarbara\\nbridge\\ncoastguard\\ncomic\\nface\\nflowers\\nforeman\\nlenna\\nman\\nmonarch\\npepper\\nppt3\\nzebra\\nAvg PSNR',\n",
       " 'Table 1: PSNR comparison on the Set14 [13] image set for upscaling of factor 2, 3 and 4',\n",
       " 'We use one Shepard interpolation layer at the top with kernel size of 8x8 and feature map number\\n16',\n",
       " 'Other configuration of the network is the same as that in our new network for inpainting',\n",
       " 'During\\ntraining, weights were randomly initialized by drawing from a Gaussian distribution with zero mean\\nand standard deviation of 0.03',\n",
       " 'AdaGrad [17] was used in all experiments with learning rate of\\n0.001 and fudge factor of 1e-6',\n",
       " 'Table 1 show the quantitative results of our ShCNN in a widely\\nused super-resolution data set [13] for upscaling images 2 times, 3 times and 4 times respectively',\n",
       " 'We compared our method with 7 methods including the two current state-of-the-art systems [11, 4]',\n",
       " 'Clear improvement over the state-of-the-art systems can be observed',\n",
       " 'Visual comparison between\\nour method and the previous methods is illustrated in figure 4 and figure 5',\n",
       " '\\n6 Conclusions\\nIn this paper, we disclosed the limitation of previous CNN architectures in image processing tasks\\nin need of translation variant interpolation',\n",
       " 'New architecture based on Shepard interpolation was\\nproposed and successfully applied to image inpainting and super-resolution',\n",
       " 'The effectiveness of\\n7',\n",
       " '\\x0c(a) Ground Truth / PSNR',\n",
       " 'Figure 5: Visual comparison',\n",
       " 'Factor 2 upscaling of the bird image in Set5 [14]',\n",
       " '\\nthe ShCNN with Shepard interpolation layers have been demonstrated by the state-of-the-art performance',\n",
       " '\\nReferences\\n[1] Krizhevsky, A., Sutskever, I., Hinton, G.E.: Imagenet classification with deep convolutional\\nneural networks',\n",
       " '(2015)\\n[3] Sun, Y., Liang, D., Wang, X., Tang, X.: Deepid3: Face recognition with very deep neural\\nnetworks',\n",
       " '(2015)\\n[4] Dong, C., Loy, C.C., He, K., , Tang, X.: Learning a deep convolutional network for image\\nsuper-resolution',\n",
       " '(2012)\\n[6] Burger, H.C., Schuler, C.J., Harmeling, S.:\\ncompete with bm3d? In: CVPR',\n",
       " '(2014)\\n[8] Eigen, D., Krishnan, D., Fergus, R.: Restoring an image taken through a window covered with\\ndirt or rain',\n",
       " '(2015)\\n[10] Shepard, D.: A two-dimensional interpolation function for irregularly-spaced data',\n",
       " 'In: 23rd\\nACM national conference',\n",
       " '(1968)\\n[11] Timofte, R., Smet, V.D., Gool, L.V.: A+: Adjusted anchored neighborhood regression for fast\\nsuper-resolution',\n",
       " '(2014)\\n[12] LeCun, Y., Bottou, L., Bengio, Y., Haffner, P.: Gradient-based learning applied to document\\nrecognition',\n",
       " '(1998)\\n[13] Zeyde, R., Elad, M., Protter, M.: On single image scale-up using sparse-representations',\n",
       " 'Curves and Surfaces 6920 (2012) 711–730\\n8',\n",
       " '\\x0c[14] Bevilacqua, M., Roumy, A., Guillemot, C., Morel, M.L.A.: Low-complexity single-image\\nsuper-resolution based on nonnegative neighbor embedding',\n",
       " '(2004)\\n[16] Timofte, R., Smet, V.D., Gool, L.V.: Anchored neighborhood regression for fast examplebased super-resolution',\n",
       " '(2013)\\n[17] Duchi, J., Hazan, E., Singer, Y.: Adaptive subgradient methods for online learning and stochastic optimization',\n",
       " 'Journal of Machine Learning Research 12 (2011) 2121–2159']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### GOOD SENTENCES ### \n",
    "\n",
    "final_sent[IF.predict(X) == 1].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Jimmy SJ',\n",
       " 'Ren∗\\nSenseTime Group Limited\\nrensijie@sensetime.com',\n",
       " 'Li Xu\\nSenseTime Group Limited\\nxuli@sensetime.com',\n",
       " 'Qiong Yan\\nSenseTime Group Limited\\nyanqiong@sensetime.com',\n",
       " 'Wenxiu Sun\\nSenseTime Group Limited\\nsunwenxiu@sensetime.com',\n",
       " 'Project page: http://www.deeplearning.cc/shepardcnn',\n",
       " 'Burger et al',\n",
       " 'Xie et al',\n",
       " 'Xu et al',\n",
       " '(K ∗ I)p / (K ∗ M)p\\nIp',\n",
       " 'Mp = 0\\nMp = 1',\n",
       " 'where I and J are the input and output images, respectively',\n",
       " 'p indexes the image coordinates',\n",
       " 'M is\\nthe binary indicator',\n",
       " 'where n is the index of layers',\n",
       " 'j\\nin Fjn−1 index the feature maps in layer n − 1',\n",
       " 'We refer to\\n4',\n",
       " 'We can discern this from Eq',\n",
       " '1x1x128 filters are used in the fourth\\nlayer',\n",
       " 'The results of the comparisons are generated using the architecture in\\n[8]',\n",
       " '(b) Bicubic / 22.10dB',\n",
       " '(c) KSVD / 23.57dB',\n",
       " '(d) NE+LLE / 23.38dB',\n",
       " '(e) ANR / 23.52dB',\n",
       " '(f) A+ / 24.42dB',\n",
       " '(g) SRCNN / 25.07dB',\n",
       " '(h) ShCNN / 25.63dB',\n",
       " 'NE+NNLS',\n",
       " 'NE+LLE',\n",
       " '24.86dB\\n28.00dB\\n26.58dB\\n29.12dB\\n26.46dB\\n34.83dB\\n30.37dB\\n34.14dB\\n34.70dB\\n29.25dB\\n32.94dB\\n34.97dB\\n26.87dB\\n30.63dB\\n30.23dB',\n",
       " '25.47dB\\n28.70dB\\n27.55dB\\n30.41dB\\n27.89 dB\\n35.57 dB\\n32.28 dB\\n36.18 dB\\n36.21 dB\\n30.44 dB\\n35.75 dB\\n36.59 dB\\n29.30 dB\\n33.21dB\\n31.81dB',\n",
       " '25.40dB\\n28.56dB\\n27.38dB\\n30.23dB\\n27.61dB\\n35.46dB\\n31.93dB\\n35.93dB\\n36.00dB\\n30.29dB\\n35.26dB\\n36.18dB\\n28.98dB\\n32.59dB\\n31.55dB',\n",
       " '25.52dB\\n28.63dB\\n27.51dB\\n30.38dB\\n27.72dB\\n35.61dB\\n32.19dB\\n36.41dB\\n36.30dB\\n30.43dB\\n35.58dB\\n36.36dB\\n28.97dB\\n33.00dB\\n31.76dB',\n",
       " 'NE+NNLS',\n",
       " 'NE+LLE',\n",
       " '23.21dB\\n26.25dB\\n24.40dB\\n26.55dB\\n23.12dB\\n32.82dB\\n27.23dB\\n31.18dB\\n31.68dB\\n27.01dB\\n29.43dB\\n32.39dB\\n23.71dB\\n26.63dB\\n27.54dB',\n",
       " '23.52dB\\n26.76dB\\n25.02dB\\n27.15dB\\n23.96dB\\n33.53dB\\n28.43dB\\n33.19dB\\n33.00dB\\n27.90dB\\n31.10dB\\n34.07dB\\n25.23dB\\n28.49dB\\n28.67dB',\n",
       " '23.49dB\\n26.67dB\\n24.86dB\\n27.00dB\\n23.83dB\\n33.45dB\\n28.21dB\\n32.87dB\\n32.82dB\\n27.72dB\\n30.76dB\\n33.56dB\\n24.81dB\\n28.12dB\\n28.44dB',\n",
       " '23.55dB\\n26.74dB\\n24.98dB\\n27.07dB\\n23.98dB\\n33.56dB\\n28.38dB\\n33.21dB\\n33.01dB\\n27.87dB\\n30.95dB\\n33.80dB\\n24.94dB\\n28.31dB\\n28.60dB',\n",
       " 'NE+NNLS',\n",
       " 'NE+LLE',\n",
       " '22.44dB\\n25.15dB\\n23.15dB\\n25.48dB\\n21.69dB\\n31.55dB\\n25.52dB\\n29.41dB\\n29.84dB\\n25.70dB\\n27.46dB\\n30.60dB\\n21.98dB\\n24.08dB\\n26.00dB',\n",
       " '22.66dB\\n25.58dB\\n23.65dB\\n25.81dB\\n22.31dB\\n32.18dB\\n26.44dB\\n31.01dB\\n30.92dB\\n26.46dB\\n28.72dB\\n32.13dB\\n23.05dB\\n25.47dB\\n26.88dB',\n",
       " '22.63dB\\n25.53dB\\n23.54dB\\n25.82dB\\n22.19dB\\n32.09dB\\n26.28dB\\n30.90dB\\n30.82dB\\n26.30dB\\n28.48dB\\n31.78dB\\n22.61dB\\n25.17dB\\n26.72dB',\n",
       " '22.67dB\\n25.58dB\\n23.60dB\\n25.81dB\\n22.26dB\\n32.19dB\\n26.38dB\\n30.90dB\\n30.93dB\\n26.38dB\\n28.58dB\\n31.87dB\\n22.77dB\\n25.36dB\\n26.81dB',\n",
       " '25.54dB\\n28.59dB\\n27.54dB\\n30.44dB\\n27.80dB\\n35.63dB\\n32.29dB\\n36.40dB\\n36.32dB\\n30.47dB\\n35.71dB\\n36.39dB\\n28.97dB\\n33.07dB\\n31.80dB',\n",
       " '25.65dB\\n28.70dB\\n27.78dB\\n30.57dB\\n28.65dB\\n35.74dB\\n33.02dB\\n36.94dB\\n36.60dB\\n30.87dB\\n37.01dB\\n37.02dB\\n30.09dB\\n33.59dB\\n32.28dB',\n",
       " '23.56dB\\n26.69dB\\n25.01dB\\n27.08dB\\n24.04dB\\n33.62dB\\n28.49dB\\n33.23dB\\n33.08dB\\n27.92dB\\n31.09dB\\n33.82dB\\n25.03dB\\n28.43dB\\n28.65dB',\n",
       " '23.62dB\\n26.47dB\\n25.17dB\\n27.27dB\\n24.38dB\\n33.76dB\\n29.05dB\\n34.30dB\\n33.52dB\\n28.28dB\\n32.14dB\\n34.74dB\\n26.09dB\\n28.98dB\\n29.13dB',\n",
       " '22.69dB\\n25.60dB\\n23.63dB\\n25.80dB\\n22.33dB\\n32.23dB\\n26.47dB\\n30.83dB\\n30.99dB\\n26.43dB\\n28.70dB\\n31.93dB\\n22.85dB\\n25.47dB\\n26.85dB',\n",
       " '22.74dB\\n25.74dB\\n23.77dB\\n25.98dB\\n22.59dB\\n32.44dB\\n26.90dB\\n32.24dB\\n31.41dB\\n26.78dB\\n29.39dB\\n32.87dB\\n23.64dB\\n25.94dB\\n27.32dB',\n",
       " '25.62dB\\n28.59dB\\n27.70dB\\n30.49dB\\n28.27dB\\n35.61dB\\n33.03dB\\n36.20dB\\n36.50dB\\n30.82dB\\n37.18dB\\n36.75dB\\n30.40dB\\n33.29dB\\n32.18dB',\n",
       " '25.79dB\\n28.59dB\\n27.92dB\\n30.82dB\\n28.70dB\\n35.75dB\\n33.53dB\\n36.14dB\\n36.71dB\\n31.06dB\\n38.09dB\\n37.03dB\\n31.07dB\\n33.51dB\\n32.48dB',\n",
       " '23.60dB\\n26.66dB\\n25.07dB\\n27.20dB\\n24.39dB\\n33.58dB\\n28.97dB\\n33.35dB\\n33.39dB\\n28.18dB\\n32.39dB\\n34.35dB\\n26.02dB\\n28.87dB\\n29.00dB',\n",
       " '23.69dB\\n26.54dB\\n25.28dB\\n27.43dB\\n24.70dB\\n33.71dB\\n29.42dB\\n34.45dB\\n33.68dB\\n28.41dB\\n33.37dB\\n34.77dB\\n26.89dB\\n29.10dB\\n29.39dB',\n",
       " '22.70dB\\n25.70dB\\n23.66dB\\n25.93dB\\n22.53dB\\n32.12dB\\n26.84dB\\n31.47dB\\n31.20dB\\n26.65dB\\n29.89dB\\n32.34dB\\n23.84dB\\n25.97dB\\n27.20dB',\n",
       " '22.75dB\\n25.80dB\\n23.83dB\\n26.13dB\\n22.74dB\\n32.35dB\\n27.18dB\\n32.30dB\\n31.45dB\\n26.82dB\\n30.30dB\\n32.82dB\\n24.49dB\\n26.21dB\\n27.51dB',\n",
       " 'Methods compared:\\nBicubic, K-SVD [13], NE+NNLS [14], NE+LLE [15], ANR [16], A+ [11], SRCNN [4], Our ShCNN',\n",
       " '(b) Bicubic / 36.81dB',\n",
       " '(c) KSVD / 39.93dB',\n",
       " '(d) NE+LLE / 40.00dB',\n",
       " '(e) ANR / 40.04dB',\n",
       " '(f) A+ / 41.12dB',\n",
       " '(g) SRCNN / 40.64dB',\n",
       " '(h) ShCNN / 41.30dB',\n",
       " 'In: NIPS',\n",
       " '(2012) 1106–1114\\n[2] Szegedy, C., Liu, W., Jia, Y., Sermanet, P., Reed, S., Anguelov, D., Erhan, D., Vanhoucke, V.,\\nRabinovich, A.: Going deeper with convolutions',\n",
       " '(2014)\\n[5] Xie, J., Xu, L., Chen, E.: Image denoising and inpainting with deep neural networks',\n",
       " 'In:\\nNIPS',\n",
       " 'Image denoising: Can plain neural networks',\n",
       " '[7] Xu, L., Ren, J.S., Liu, C., Jia, J.: Deep convolutional neural network for image deconvolution',\n",
       " 'In: NIPS',\n",
       " '(2013)\\n[9] Xu, L., Ren, J.S., Yan, Q., Liao, R., Jia, J.: Deep edge-aware filters',\n",
       " 'In: Proceedings of IEEE',\n",
       " '(2012)\\n[15] Chang, H., Yeung, D.Y., Xiong, Y.: Super-resolution through neighbor embedding']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### BAD SENTENCES (ANOMALIES) ###\n",
    "\n",
    "final_sent[IF.predict(X) == -1].tolist()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
